r/guest_worker
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Compare commits

base: r:master
Branches Tags
r:master
r:assignment
..
compare: r:assignment
Branches Tags
r:master
r:assignment

No commits in common. "master" and "assignment" have entirely different histories.
master ... assignment

37 changed files with 518 additions and 1629 deletions
Show stats Expand all files Collapse all files

4
.gitignore vendored
View file

@ -5,8 +5,6 @@ node_modules
.project .project
config.local.yml config.local.yml
hit_store.db hit_store.db
backup/*
www/scans/*.svg www/scans/*.svg
#scanimation/interfa #scanimation/interfa
GeoLite2-Country.mmdb

2
Pipfile
View file

@ -17,8 +17,6 @@ Pillow = "*"
tqdm = "*" tqdm = "*"
serial = "*" serial = "*"
pyserial = "*" pyserial = "*"
country-converter = "*"
svgpathtools = "*"
[dev-packages] [dev-packages]

119
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "d1d2c21498d8c771cdd3233e304f260e7fe6020dfb6a1529286e85e8ead0f34f" "sha256": "3c0954c7917f567561faffcf18031aa0718c5144698d0de7022dd9ad9d49b1c4"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -21,18 +21,18 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:5222edc5b20d5c6ab7440fc4f89f987ead05be37ff5cc5359a3b9148d9b5a51e", "sha256:98fdd6fa754e17c0d9e87fbb464c43c7e72aaa6b4f78b418eba47b7d15deffee",
"sha256:bd3337cfc15613b0091fa567dc3065d94df88e5837ba1adbb1e35b91db728a66" "sha256:fdaae8edd63ae114107d375862069d17de23e00489a65169b8141ddee6bdf78b"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.11.7" "version": "==1.10.48"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:9a17d36ee43f1398c7db3cb29aa2216de94bcb60f058b1c645d71e72a330ddf8", "sha256:29370f50af7870661609fbfbc4ed01ef2fd531b87b98729700526d1a4b3a2f89",
"sha256:e4b82b1a7389f3d16732eb839240c9d3e42470100d5a71415ea2a0a35b911b23" "sha256:7f60edf33c6f5b7c1c9b9377267bdc56495f52704607f713d4c3bd1d82a08334"
], ],
"version": "==1.14.7" "version": "==1.13.48"
}, },
"certifi": { "certifi": {
"hashes": [ "hashes": [
@ -56,13 +56,6 @@
"index": "pypi", "index": "pypi",
"version": "==10.0" "version": "==10.0"
}, },
"country-converter": {
"hashes": [
"sha256:bc01ba2592b77a78b4f3e6f76600ca27852d71d1512cf1f320fecbcaaea3c6f9"
],
"index": "pypi",
"version": "==0.6.7"
},
"docutils": { "docutils": {
"hashes": [ "hashes": [
"sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
@ -166,56 +159,6 @@
], ],
"version": "==1.5.2" "version": "==1.5.2"
}, },
"numpy": {
"hashes": [
"sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6",
"sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e",
"sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc",
"sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc",
"sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a",
"sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa",
"sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3",
"sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121",
"sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971",
"sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26",
"sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd",
"sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480",
"sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec",
"sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77",
"sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57",
"sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07",
"sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572",
"sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73",
"sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca",
"sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474",
"sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5"
],
"version": "==1.18.1"
},
"pandas": {
"hashes": [
"sha256:00dff3a8e337f5ed7ad295d98a31821d3d0fe7792da82d78d7fd79b89c03ea9d",
"sha256:22361b1597c8c2ffd697aa9bf85423afa9e1fcfa6b1ea821054a244d5f24d75e",
"sha256:255920e63850dc512ce356233081098554d641ba99c3767dde9e9f35630f994b",
"sha256:26382aab9c119735908d94d2c5c08020a4a0a82969b7e5eefb92f902b3b30ad7",
"sha256:33970f4cacdd9a0ddb8f21e151bfb9f178afb7c36eb7c25b9094c02876f385c2",
"sha256:4545467a637e0e1393f7d05d61dace89689ad6d6f66f267f86fff737b702cce9",
"sha256:52da74df8a9c9a103af0a72c9d5fdc8e0183a90884278db7f386b5692a2220a4",
"sha256:61741f5aeb252f39c3031d11405305b6d10ce663c53bc3112705d7ad66c013d0",
"sha256:6a3ac2c87e4e32a969921d1428525f09462770c349147aa8e9ab95f88c71ec71",
"sha256:7458c48e3d15b8aaa7d575be60e1e4dd70348efcd9376656b72fecd55c59a4c3",
"sha256:78bf638993219311377ce9836b3dc05f627a666d0dbc8cec37c0ff3c9ada673b",
"sha256:8153705d6545fd9eb6dd2bc79301bff08825d2e2f716d5dced48daafc2d0b81f",
"sha256:975c461accd14e89d71772e89108a050fa824c0b87a67d34cedf245f6681fc17",
"sha256:9962957a27bfb70ab64103d0a7b42fa59c642fb4ed4cb75d0227b7bb9228535d",
"sha256:adc3d3a3f9e59a38d923e90e20c4922fc62d1e5a03d083440468c6d8f3f1ae0a",
"sha256:bbe3eb765a0b1e578833d243e2814b60c825b7fdbf4cdfe8e8aae8a08ed56ecf",
"sha256:df8864824b1fe488cf778c3650ee59c3a0d8f42e53707de167ba6b4f7d35f133",
"sha256:e45055c30a608076e31a9fcd780a956ed3b1fa20db61561b8d88b79259f526f7",
"sha256:ee50c2142cdcf41995655d499a157d0a812fce55c97d9aad13bc1eef837ed36c"
],
"version": "==0.25.3"
},
"pillow": { "pillow": {
"hashes": [ "hashes": [
"sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be", "sha256:0a628977ac2e01ca96aaae247ec2bd38e729631ddf2221b4b715446fd45505be",
@ -244,13 +187,6 @@
"index": "pypi", "index": "pypi",
"version": "==7.0.0" "version": "==7.0.0"
}, },
"pyparsing": {
"hashes": [
"sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f",
"sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec"
],
"version": "==2.4.6"
},
"pyserial": { "pyserial": {
"hashes": [ "hashes": [
"sha256:6e2d401fdee0eab996cf734e67773a0143b932772ca8b42451440cfed942c627", "sha256:6e2d401fdee0eab996cf734e67773a0143b932772ca8b42451440cfed942c627",
@ -264,6 +200,7 @@
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
], ],
"markers": "python_version >= '2.7'",
"version": "==2.8.1" "version": "==2.8.1"
}, },
"python-magic": { "python-magic": {
@ -274,13 +211,6 @@
"index": "pypi", "index": "pypi",
"version": "==0.4.15" "version": "==0.4.15"
}, },
"pytz": {
"hashes": [
"sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d",
"sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be"
],
"version": "==2019.3"
},
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
"sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6", "sha256:059b2ee3194d718896c0ad077dd8c043e5e909d9180f387ce42012662a4946d6",
@ -307,10 +237,10 @@
}, },
"s3transfer": { "s3transfer": {
"hashes": [ "hashes": [
"sha256:248dffd2de2dfb870c507b412fc22ed37cd3255293e293c395158e7c55fbe5f9", "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d",
"sha256:80ed96731b3bd77395cd6197246069092015e1124164b2c152c8f741a823dd04" "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba"
], ],
"version": "==0.3.1" "version": "==0.2.1"
}, },
"serial": { "serial": {
"hashes": [ "hashes": [
@ -322,10 +252,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
], ],
"version": "==1.14.0" "version": "==1.13.0"
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
@ -334,21 +264,6 @@
"index": "pypi", "index": "pypi",
"version": "==1.3.12" "version": "==1.3.12"
}, },
"svgpathtools": {
"hashes": [
"sha256:7f7bdafe2c03b312178460104705e1d554d8cf36c898bec41bdce9fed3504746",
"sha256:e4b3784ae41b725fbce6a33a8981210967b16d0b557cb5d98c0ed0c81f0f89b9"
],
"index": "pypi",
"version": "==1.3.3"
},
"svgwrite": {
"hashes": [
"sha256:11e47749b159ed7004721e11d380b4642a26154b8cb2f7b0102fea9c71a3dfa1",
"sha256:50fec23dc3fd49103808f0d672124f8c573ec5899da5686df734f856b8d3b737"
],
"version": "==1.3.1"
},
"tornado": { "tornado": {
"hashes": [ "hashes": [
"sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c", "sha256:349884248c36801afa19e342a77cc4458caca694b0eda633f5878e458a44cb2c",
@ -372,10 +287,10 @@
}, },
"urllib3": { "urllib3": {
"hashes": [ "hashes": [
"sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc", "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
"sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc" "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
], ],
"version": "==1.25.8" "version": "==1.25.7"
} }
}, },
"develop": {} "develop": {}

36
backup_and_reset.sh
View file

@ -1,36 +0,0 @@
#!/bin/bash
BACKUP_DIR="backup/$1"
if [ -d "$BACKUP_DIR" ]; then
>&2 echo "$BACKUP_DIR already exists. (usage: ./backup_and_reset.sh BACKUP_NAME)"
exit 1
fi
read -p "If you continue current config will be copied to '$BACKUP_DIR'. Continue? [y/n]" -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo "Cancelling"
exit 1
fi
mkdir "$BACKUP_DIR"
cp www/scans "$BACKUP_DIR/" -Rv
cp hit_store.db "$BACKUP_DIR/"
cp scanimation/interfaces/frames "$BACKUP_DIR/" -Rv
read -p "Reset or keep original files. WARNING, type yes for reset [y/n]" -r
echo # (optional) move to a new line
if [[ ! $REPLY =~ ^[Yy]$ ]]
then
echo "Exit without resetting."
exit 1
fi
# reset
rm hit_store.db
rm scanimation/interfaces/frames/*.jpg
rm www/scans/*.svg
echo "Done resetting files."

14
scanimation.service
View file

@ -1,14 +0,0 @@
[Unit]
Description=GuestWorker Scanimation server
After=network.target
[Service]
WorkingDirectory=/home/mt/guest_worker/scanimation
ExecStart=/usr/bin/node run.js
User=mt
Restart=Always
RestartSec=10
[Install]
WantedBy=multi-user.target

2
scanimation/LocalServer.js
View file

@ -31,7 +31,6 @@ let LocalServer = (config) => {
socket.on('animationInit', function(){ socket.on('animationInit', function(){
fs.readdir(config.frames_folder, (err, files) => { fs.readdir(config.frames_folder, (err, files) => {
files = files.filter(file => file.endsWith('.jpg'))
socket.emit('frameData', {frames: files}) socket.emit('frameData', {frames: files})
console.log('starting animation') console.log('starting animation')
}) })
@ -39,7 +38,6 @@ let LocalServer = (config) => {
watch(config.frames_folder, function(e, name){ watch(config.frames_folder, function(e, name){
fs.readdir(config.frames_folder, (err, files) => { fs.readdir(config.frames_folder, (err, files) => {
files = files.filter(file => file.endsWith('.jpg'))
socket.emit('frameData', {frames: files}) socket.emit('frameData', {frames: files})
console.log(`frames changed => ${name}`) console.log(`frames changed => ${name}`)
}) })

138
scanimation/package-lock.json generated
View file

@ -9,7 +9,7 @@
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==", "integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
"requires": { "requires": {
"mime-types": "2.1.24", "mime-types": "~2.1.24",
"negotiator": "0.6.2" "negotiator": "0.6.2"
} }
}, },
@ -67,15 +67,15 @@
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==", "integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
"requires": { "requires": {
"bytes": "3.1.0", "bytes": "3.1.0",
"content-type": "1.0.4", "content-type": "~1.0.4",
"debug": "2.6.9", "debug": "2.6.9",
"depd": "1.1.2", "depd": "~1.1.2",
"http-errors": "1.7.2", "http-errors": "1.7.2",
"iconv-lite": "0.4.24", "iconv-lite": "0.4.24",
"on-finished": "2.3.0", "on-finished": "~2.3.0",
"qs": "6.7.0", "qs": "6.7.0",
"raw-body": "2.4.0", "raw-body": "2.4.0",
"type-is": "1.6.18" "type-is": "~1.6.17"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -146,7 +146,7 @@
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": { "requires": {
"ms": "2.1.2" "ms": "^2.1.1"
} }
}, },
"depd": { "depd": {
@ -174,12 +174,12 @@
"resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.4.0.tgz", "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.4.0.tgz",
"integrity": "sha512-XCyYVWzcHnK5cMz7G4VTu2W7zJS7SM1QkcelghyIk/FmobWBtXE7fwhBusEKvCSqc3bMh8fNFMlUkCKTFRxH2w==", "integrity": "sha512-XCyYVWzcHnK5cMz7G4VTu2W7zJS7SM1QkcelghyIk/FmobWBtXE7fwhBusEKvCSqc3bMh8fNFMlUkCKTFRxH2w==",
"requires": { "requires": {
"accepts": "1.3.7", "accepts": "~1.3.4",
"base64id": "2.0.0", "base64id": "2.0.0",
"cookie": "0.3.1", "cookie": "0.3.1",
"debug": "4.1.1", "debug": "~4.1.0",
"engine.io-parser": "2.2.0", "engine.io-parser": "~2.2.0",
"ws": "7.1.2" "ws": "^7.1.2"
} }
}, },
"engine.io-client": { "engine.io-client": {
@ -189,14 +189,14 @@
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"component-inherit": "0.0.3", "component-inherit": "0.0.3",
"debug": "4.1.1", "debug": "~4.1.0",
"engine.io-parser": "2.2.0", "engine.io-parser": "~2.2.0",
"has-cors": "1.1.0", "has-cors": "1.1.0",
"indexof": "0.0.1", "indexof": "0.0.1",
"parseqs": "0.0.5", "parseqs": "0.0.5",
"parseuri": "0.0.5", "parseuri": "0.0.5",
"ws": "6.1.4", "ws": "~6.1.0",
"xmlhttprequest-ssl": "1.5.5", "xmlhttprequest-ssl": "~1.5.4",
"yeast": "0.1.2" "yeast": "0.1.2"
}, },
"dependencies": { "dependencies": {
@ -205,7 +205,7 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
"integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==", "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
"requires": { "requires": {
"async-limiter": "1.0.1" "async-limiter": "~1.0.0"
} }
} }
} }
@ -216,10 +216,10 @@
"integrity": "sha512-6I3qD9iUxotsC5HEMuuGsKA0cXerGz+4uGcXQEkfBidgKf0amsjrrtwcbwK/nzpZBxclXlV7gGl9dgWvu4LF6w==", "integrity": "sha512-6I3qD9iUxotsC5HEMuuGsKA0cXerGz+4uGcXQEkfBidgKf0amsjrrtwcbwK/nzpZBxclXlV7gGl9dgWvu4LF6w==",
"requires": { "requires": {
"after": "0.8.2", "after": "0.8.2",
"arraybuffer.slice": "0.0.7", "arraybuffer.slice": "~0.0.7",
"base64-arraybuffer": "0.1.5", "base64-arraybuffer": "0.1.5",
"blob": "0.0.5", "blob": "0.0.5",
"has-binary2": "1.0.3" "has-binary2": "~1.0.2"
} }
}, },
"escape-html": { "escape-html": {
@ -237,36 +237,36 @@
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz", "resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==", "integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
"requires": { "requires": {
"accepts": "1.3.7", "accepts": "~1.3.7",
"array-flatten": "1.1.1", "array-flatten": "1.1.1",
"body-parser": "1.19.0", "body-parser": "1.19.0",
"content-disposition": "0.5.3", "content-disposition": "0.5.3",
"content-type": "1.0.4", "content-type": "~1.0.4",
"cookie": "0.4.0", "cookie": "0.4.0",
"cookie-signature": "1.0.6", "cookie-signature": "1.0.6",
"debug": "2.6.9", "debug": "2.6.9",
"depd": "1.1.2", "depd": "~1.1.2",
"encodeurl": "1.0.2", "encodeurl": "~1.0.2",
"escape-html": "1.0.3", "escape-html": "~1.0.3",
"etag": "1.8.1", "etag": "~1.8.1",
"finalhandler": "1.1.2", "finalhandler": "~1.1.2",
"fresh": "0.5.2", "fresh": "0.5.2",
"merge-descriptors": "1.0.1", "merge-descriptors": "1.0.1",
"methods": "1.1.2", "methods": "~1.1.2",
"on-finished": "2.3.0", "on-finished": "~2.3.0",
"parseurl": "1.3.3", "parseurl": "~1.3.3",
"path-to-regexp": "0.1.7", "path-to-regexp": "0.1.7",
"proxy-addr": "2.0.5", "proxy-addr": "~2.0.5",
"qs": "6.7.0", "qs": "6.7.0",
"range-parser": "1.2.1", "range-parser": "~1.2.1",
"safe-buffer": "5.1.2", "safe-buffer": "5.1.2",
"send": "0.17.1", "send": "0.17.1",
"serve-static": "1.14.1", "serve-static": "1.14.1",
"setprototypeof": "1.1.1", "setprototypeof": "1.1.1",
"statuses": "1.5.0", "statuses": "~1.5.0",
"type-is": "1.6.18", "type-is": "~1.6.18",
"utils-merge": "1.0.1", "utils-merge": "1.0.1",
"vary": "1.1.2" "vary": "~1.1.2"
}, },
"dependencies": { "dependencies": {
"cookie": { "cookie": {
@ -295,12 +295,12 @@
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==", "integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
"requires": { "requires": {
"debug": "2.6.9", "debug": "2.6.9",
"encodeurl": "1.0.2", "encodeurl": "~1.0.2",
"escape-html": "1.0.3", "escape-html": "~1.0.3",
"on-finished": "2.3.0", "on-finished": "~2.3.0",
"parseurl": "1.3.3", "parseurl": "~1.3.3",
"statuses": "1.5.0", "statuses": "~1.5.0",
"unpipe": "1.0.0" "unpipe": "~1.0.0"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -346,10 +346,10 @@
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==", "integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
"requires": { "requires": {
"depd": "1.1.2", "depd": "~1.1.2",
"inherits": "2.0.3", "inherits": "2.0.3",
"setprototypeof": "1.1.1", "setprototypeof": "1.1.1",
"statuses": "1.5.0", "statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0" "toidentifier": "1.0.0"
} }
}, },
@ -358,7 +358,7 @@
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": { "requires": {
"safer-buffer": "2.1.2" "safer-buffer": ">= 2.1.2 < 3"
} }
}, },
"indexof": { "indexof": {
@ -447,7 +447,7 @@
"resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz", "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz",
"integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=", "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=",
"requires": { "requires": {
"better-assert": "1.0.2" "better-assert": "~1.0.0"
} }
}, },
"parseuri": { "parseuri": {
@ -455,7 +455,7 @@
"resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz", "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz",
"integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=", "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=",
"requires": { "requires": {
"better-assert": "1.0.2" "better-assert": "~1.0.0"
} }
}, },
"parseurl": { "parseurl": {
@ -473,7 +473,7 @@
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.5.tgz",
"integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==", "integrity": "sha512-t/7RxHXPH6cJtP0pRG6smSr9QJidhB+3kXu0KgXnbGYMgzEnUxRQ4/LDdfOwZEMyIh3/xHb8PX3t+lfL9z+YVQ==",
"requires": { "requires": {
"forwarded": "0.1.2", "forwarded": "~0.1.2",
"ipaddr.js": "1.9.0" "ipaddr.js": "1.9.0"
} }
}, },
@ -519,18 +519,18 @@
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==", "integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
"requires": { "requires": {
"debug": "2.6.9", "debug": "2.6.9",
"depd": "1.1.2", "depd": "~1.1.2",
"destroy": "1.0.4", "destroy": "~1.0.4",
"encodeurl": "1.0.2", "encodeurl": "~1.0.2",
"escape-html": "1.0.3", "escape-html": "~1.0.3",
"etag": "1.8.1", "etag": "~1.8.1",
"fresh": "0.5.2", "fresh": "0.5.2",
"http-errors": "1.7.2", "http-errors": "~1.7.2",
"mime": "1.6.0", "mime": "1.6.0",
"ms": "2.1.1", "ms": "2.1.1",
"on-finished": "2.3.0", "on-finished": "~2.3.0",
"range-parser": "1.2.1", "range-parser": "~1.2.1",
"statuses": "1.5.0" "statuses": "~1.5.0"
}, },
"dependencies": { "dependencies": {
"debug": { "debug": {
@ -560,9 +560,9 @@
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==", "integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
"requires": { "requires": {
"encodeurl": "1.0.2", "encodeurl": "~1.0.2",
"escape-html": "1.0.3", "escape-html": "~1.0.3",
"parseurl": "1.3.3", "parseurl": "~1.3.3",
"send": "0.17.1" "send": "0.17.1"
} }
}, },
@ -576,12 +576,12 @@
"resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.3.0.tgz", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.3.0.tgz",
"integrity": "sha512-2A892lrj0GcgR/9Qk81EaY2gYhCBxurV0PfmmESO6p27QPrUK1J3zdns+5QPqvUYK2q657nSj0guoIil9+7eFg==", "integrity": "sha512-2A892lrj0GcgR/9Qk81EaY2gYhCBxurV0PfmmESO6p27QPrUK1J3zdns+5QPqvUYK2q657nSj0guoIil9+7eFg==",
"requires": { "requires": {
"debug": "4.1.1", "debug": "~4.1.0",
"engine.io": "3.4.0", "engine.io": "~3.4.0",
"has-binary2": "1.0.3", "has-binary2": "~1.0.2",
"socket.io-adapter": "1.1.1", "socket.io-adapter": "~1.1.0",
"socket.io-client": "2.3.0", "socket.io-client": "2.3.0",
"socket.io-parser": "3.4.0" "socket.io-parser": "~3.4.0"
} }
}, },
"socket.io-adapter": { "socket.io-adapter": {
@ -598,15 +598,15 @@
"base64-arraybuffer": "0.1.5", "base64-arraybuffer": "0.1.5",
"component-bind": "1.0.0", "component-bind": "1.0.0",
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "4.1.1", "debug": "~4.1.0",
"engine.io-client": "3.4.0", "engine.io-client": "~3.4.0",
"has-binary2": "1.0.3", "has-binary2": "~1.0.2",
"has-cors": "1.1.0", "has-cors": "1.1.0",
"indexof": "0.0.1", "indexof": "0.0.1",
"object-component": "0.0.3", "object-component": "0.0.3",
"parseqs": "0.0.5", "parseqs": "0.0.5",
"parseuri": "0.0.5", "parseuri": "0.0.5",
"socket.io-parser": "3.3.0", "socket.io-parser": "~3.3.0",
"to-array": "0.1.4" "to-array": "0.1.4"
}, },
"dependencies": { "dependencies": {
@ -621,7 +621,7 @@
"integrity": "sha512-hczmV6bDgdaEbVqhAeVMM/jfUfzuEZHsQg6eOmLgJht6G3mPKMxYm75w2+qhAQZ+4X+1+ATZ+QFKeOZD5riHng==", "integrity": "sha512-hczmV6bDgdaEbVqhAeVMM/jfUfzuEZHsQg6eOmLgJht6G3mPKMxYm75w2+qhAQZ+4X+1+ATZ+QFKeOZD5riHng==",
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "3.1.0", "debug": "~3.1.0",
"isarray": "2.0.1" "isarray": "2.0.1"
}, },
"dependencies": { "dependencies": {
@ -643,7 +643,7 @@
"integrity": "sha512-/G/VOI+3DBp0+DJKW4KesGnQkQPFmUCbA/oO2QGT6CWxU7hLGWqU3tyuzeSK/dqcyeHsQg1vTe9jiZI8GU9SCQ==", "integrity": "sha512-/G/VOI+3DBp0+DJKW4KesGnQkQPFmUCbA/oO2QGT6CWxU7hLGWqU3tyuzeSK/dqcyeHsQg1vTe9jiZI8GU9SCQ==",
"requires": { "requires": {
"component-emitter": "1.2.1", "component-emitter": "1.2.1",
"debug": "4.1.1", "debug": "~4.1.0",
"isarray": "2.0.1" "isarray": "2.0.1"
} }
}, },
@ -668,7 +668,7 @@
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
"requires": { "requires": {
"media-typer": "0.3.0", "media-typer": "0.3.0",
"mime-types": "2.1.24" "mime-types": "~2.1.24"
} }
}, },
"unpipe": { "unpipe": {
@ -691,7 +691,7 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-7.1.2.tgz", "resolved": "https://registry.npmjs.org/ws/-/ws-7.1.2.tgz",
"integrity": "sha512-gftXq3XI81cJCgkUiAVixA0raD9IVmXqsylCrjRygw4+UOOGzPoxnQ6r/CnVL9i+mDncJo94tSkyrtuuQVBmrg==", "integrity": "sha512-gftXq3XI81cJCgkUiAVixA0raD9IVmXqsylCrjRygw4+UOOGzPoxnQ6r/CnVL9i+mDncJo94tSkyrtuuQVBmrg==",
"requires": { "requires": {
"async-limiter": "1.0.1" "async-limiter": "^1.0.0"
} }
}, },
"xmlhttprequest-ssl": { "xmlhttprequest-ssl": {

7
sorteerhoed.py
View file

@ -21,11 +21,6 @@ if __name__ == '__main__':
action='store_true', action='store_true',
help='Skip attempt to connect to plotter' help='Skip attempt to connect to plotter'
) )
argParser.add_argument(
'--autostart',
action='store_true',
help='Don\'t require a visit to the control panel to start the first hit. Usefull when you know plotter & scanner are already set up'
)
argParser.add_argument( argParser.add_argument(
'--for-real', '--for-real',
action='store_true', action='store_true',
@ -44,7 +39,7 @@ if __name__ == '__main__':
coloredlogs.install( coloredlogs.install(
level=loglevel, level=loglevel,
# default: "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s" # default: "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s"
fmt="%(asctime)s %(hostname)s %(name)s[%(process)d,%(threadName)s,%(pathname)s:%(lineno)d] %(levelname)s %(message)s" fmt="%(asctime)s %(hostname)s %(name)s[%(process)d,%(threadName)s] %(levelname)s %(message)s"
) )
# File logging # File logging

14
sorteerhoed.service
View file

@ -1,14 +0,0 @@
[Unit]
Description=GuestWorker Main server
After=network.target
[Service]
WorkingDirectory=/home/mt/guest_worker
ExecStart=/usr/bin/pipenv run python sorteerhoed.py --config config.local.yml
User=mt
Restart=Always
RestartSec=10
[Install]
WantedBy=multi-user.target

240
sorteerhoed/HITStore.py
View file

@ -4,19 +4,19 @@ from sqlalchemy import Column, Integer, String, DateTime, Float
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from sqlalchemy.sql.schema import ForeignKey, Sequence from sqlalchemy.sql.schema import ForeignKey, Sequence
from sqlalchemy.engine import create_engine from sqlalchemy.engine import create_engine
from sqlalchemy.orm.session import sessionmaker, object_session from sqlalchemy.orm.session import sessionmaker
import datetime import datetime
from contextlib import contextmanager from contextlib import contextmanager
import uuid import uuid
import os import os
import country_converter import coloredlogs
from svgpathtools import svg2paths import argparse
from sqlalchemy.sql.functions import func
mainLogger = logging.getLogger("sorteerhoed") mainLogger = logging.getLogger("sorteerhoed")
logger = mainLogger.getChild("store") logger = mainLogger.getChild("store")
Base = declarative_base() Base = declarative_base()
cc = country_converter.CountryConverter()
""" """
HIT lifetime: HIT lifetime:
@ -43,7 +43,6 @@ class HIT(Base):
created_at = Column(DateTime, default=datetime.datetime.utcnow) created_at = Column(DateTime, default=datetime.datetime.utcnow)
updated_at = Column(DateTime, default=datetime.datetime.utcnow) updated_at = Column(DateTime, default=datetime.datetime.utcnow)
scanned_at = Column(DateTime, default=None) scanned_at = Column(DateTime, default=None)
plotted_at = Column(DateTime, default=None)
deleted_at = Column(DateTime, default=None) deleted_at = Column(DateTime, default=None)
assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at") assignments = relationship("Assignment", back_populates="hit", order_by="Assignment.created_at")
fee = Column(Float(precision=2), default=None) fee = Column(Float(precision=2), default=None)
@ -55,15 +54,11 @@ class HIT(Base):
def getImagePath(self): def getImagePath(self):
return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg") return os.path.join('scanimation/interfaces/frames', f"{self.id:06d}.jpg")
def getImageUrl(self):
return os.path.join('/frames', f"{self.id:06d}.jpg")
def getSvgImageUrl(self): def getSvgImageUrl(self):
return f"/scans/{self.id:06d}.svg" return f"scans/{self.id:06d}.svg"
def getSvgImagePath(self): def getSvgImagePath(self):
# os.path.join on svgImageUrl leads to invalid absolute url return os.path.join('www', self.getSvgImageUrl())
return os.path.join(f'www/scans/{self.id:06d}.svg')
def getLastAssignment(self): def getLastAssignment(self):
if not len(self.assignments): if not len(self.assignments):
@ -73,73 +68,23 @@ class HIT(Base):
def getAssignmentById(self, assignmentId): def getAssignmentById(self, assignmentId):
for a in self.assignments: for a in self.assignments:
if a.assignment_id == assignmentId: if a.assignment_id == assignmentId:
return a return
return None
def getStatus(self): def getStatus(self):
assignment = self.getLastAssignment()
if self.deleted_at:
return "deleted"
if not self.hit_id:
return "creating"
if not assignment:
return "awaiting worker"
if self.scanned_at: if self.scanned_at:
return "scanned" return "completed"
return assignment.getStatus() if self.submit_hit_at:
return "submission confirmed"
def toDict(self) -> dict: if self.submit_page_at:
values = {c.name: getattr(self, c.name) for c in self.__table__.columns} return "submitted by worker"
assignment = self.getLastAssignment() if self.open_page_at:
values['assignment'] = assignment.toDict() if assignment else None return "working"
values['state'] = self.getStatus() if self.accept_time:
values['scan_image'] = self.getImageUrl() if self.scanned_at else None return "accepted by worker"
values['svg_image'] = self.getSvgImageUrl() if self.isSubmitted() else None # on abandon:
values['preceding_assignments'] = [a.toShortDict() for a in self.getBasedOnAssignments()] if self.worker_id:
values['preceding_assignments'].append({ return "abandoned by worker"
'worker_id': 'Ruben van de Ven & Merijn van Moll', return "awaiting worker"
'turk_country': 'the Netherlands',
'turk_country_code': 'NL'
})
if not values['svg_image'] or not os.path.exists(self.getSvgImagePath()):
values['path_length'] = None
else:
try:
paths, _ = svg2paths(self.getSvgImagePath())
values['path_length'] = round(paths[0].length())
except:
values['path_length'] = None
return values
def delete(self):
self.deleted_at = datetime.datetime.utcnow()
def isSubmitted(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.submit_page_at)
def isConfirmed(self) -> bool:
a = self.getLastAssignment()
if not a:
return False
return bool(a.confirmed_at)
def getBasedOnAssignments(self):
"""
Get preceding assignments, one per worker, excluding the one who did this HIT
"""
assignment = self.getLastAssignment()
session = object_session(self)
q = session.query(Assignment).\
filter(Assignment.submit_page_at < self.created_at).\
filter(Assignment.worker_id != None).\
group_by(Assignment.worker_id).\
order_by(Assignment.created_at.desc())
if assignment and assignment.worker_id:
q = q.filter(Assignment.worker_id != assignment.worker_id)
return q
class Assignment(Base): class Assignment(Base):
__tablename__ = 'assignments' __tablename__ = 'assignments'
@ -154,7 +99,7 @@ class Assignment(Base):
assignment_id = Column(String(255), default = None) assignment_id = Column(String(255), default = None)
worker_id = Column(String(255), default = None) worker_id = Column(String(255), default = None)
accept_at = Column(DateTime, default=None) # accept time acccording to SQS accept_at = Column(DateTime, default=None)
# open_page_at = Column(DateTime, default=None) # open_page_at = Column(DateTime, default=None)
submit_page_at = Column(DateTime, default=None) # Submit the page submit_page_at = Column(DateTime, default=None) # Submit the page
confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon confirmed_at = Column(DateTime, default=None) # validate with UUID when getting Message from Amazon
@ -164,52 +109,6 @@ class Assignment(Base):
answer = Column(String(255), default=None) answer = Column(String(255), default=None)
turk_ip = Column(String(255), default=None) turk_ip = Column(String(255), default=None)
turk_country = Column(String(255), default=None) turk_country = Column(String(255), default=None)
turk_os = Column(String(255), default=None)
turk_browser = Column(String(255), default=None)
def getStatus(self):
if self.rejected_at:
return "rejected"
if self.abandoned_at:
return "abandoned"
if not self.submit_page_at:
return "working"
if not self.confirmed_at:
return "submitted"
return "confirmed"
def toDict(self) -> dict:
values = {c.name: getattr(self, c.name) for c in self.__table__.columns}
if self.turk_country:
if self.turk_country == 'Unknown':
values['turk_country_code'] = '--'
else:
values['turk_country_code'] = cc.convert([self.turk_country], to='ISO2')
else:
values['turk_country_code'] = None
return values
def toShortDict(self) -> dict:
values = {
'worker_id': self.worker_id,
'turk_country': self.turk_country
}
if self.turk_country:
if self.turk_country == 'Unknown':
values['turk_country_code'] = '--'
else:
values['turk_country_code'] = cc.convert([self.turk_country], to='ISO2')
else:
values['turk_country_code'] = None
return values
def getOriginalAssignmentId(self):
"""
Initial assumption: assignmentId would be unique for each worker. But it is not.
Hence assignment_id now gets the value assigmentId_workerId (confusing to say the least!)
Sometimes we want to recover the assignment id from this combination
"""
return self.assignment_id.split('_')[0]
class Store: class Store:
def __init__(self, db_filename, logLevel=0): def __init__(self, db_filename, logLevel=0):
@ -217,35 +116,12 @@ class Store:
if logLevel <= logging.DEBUG: if logLevel <= logging.DEBUG:
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
needsInitialization = not os.path.exists(path)
self.engine = create_engine('sqlite:///'+path, echo=False, connect_args={'check_same_thread': False}) self.engine = create_engine('sqlite:///'+path, echo=False, connect_args={'check_same_thread': False})
Base.metadata.create_all(self.engine) Base.metadata.create_all(self.engine)
self.Session = sessionmaker(bind=self.engine) self.Session = sessionmaker(bind=self.engine)
self.session = self.Session() self.session = self.Session()
self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes self.currentHit = None # mirrors Centralmanagmenet, stored here so we can quickly access it from webserver classes
self.updateHooks = []
# if needsInitialization:
# self.insertInitialContent()
#
# def insertInitialContent(self):
# hit = self.createHIT()
# assignment = self.newAssignment(hit, 'initial')
#
def registerUpdateHook(self, hook):
if hook not in self.updateHooks:
logger.info(f"Register update hook: {hook}")
self.updateHooks.append(hook)
def triggerUpdateHooks(self, hit = None):
for hook in self.updateHooks:
if callable(hook): # it's a method
hook(hit)
else: # assume it's an object
hook.update(hit)
@contextmanager @contextmanager
def getSession(self): def getSession(self):
@ -257,8 +133,8 @@ class Store:
self.session.rollback() self.session.rollback()
raise raise
def getHits(self): def getHits(self, session):
return self.session.query(HIT).order_by(HIT.created_at.desc()) return self.session.query(Source).order_by(HIT.created_at.desc())
def getHitById(self, hitId): def getHitById(self, hitId):
return self.session.query(HIT).\ return self.session.query(HIT).\
@ -271,20 +147,8 @@ class Store:
def getLastSubmittedHit(self): def getLastSubmittedHit(self):
return self.session.query(HIT).\ return self.session.query(HIT).\
join(Assignment).\ filter(HIT.submit_page_at!=None).\
filter(Assignment.submit_page_at!=None).\ order_by(HIT.submit_page_at.desc()).first()
order_by(HIT.created_at.desc()).first()
def getNewestHits(self, n = 2) -> list:
q = self.session.query(HIT).\
filter(HIT.deleted_at==None).\
order_by(HIT.created_at.desc())
if n is not None:
q = q.limit(n)
hits = list(q)
# select DESC, because we want latest, then reverse list to get in right order
hits.reverse()
return hits
def createHIT(self) -> HIT: def createHIT(self) -> HIT:
with self.getSession() as s: with self.getSession() as s:
@ -293,57 +157,40 @@ class Store:
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created HIT {hit.id}") logger.info(f"Created HIT {hit.id}")
self.triggerUpdateHooks(hit)
return hit return hit
def newAssignment(self, hit: HIT, assignmentId) -> Assignment: def newAssignment(self, hit: HIT) -> Assignment:
# TODO: reset() central management if has pending lastAssignment()
with self.getSession() as s: with self.getSession() as s:
assignment = Assignment() assignment = Assignment()
assignment.assignment_id = assignmentId
hit.assignments.append(assignment) hit.assignments.append(assignment)
s.add(assignment) s.add(assignment)
s.flush() s.flush()
s.refresh(hit) s.refresh(hit)
logger.info(f"Created Assignment {assignment.id}") logger.info(f"Created Assignment {assignment.id}")
self.triggerUpdateHooks(hit)
return assignment return assignment
def saveHIT(self, hit): def saveHIT(self, hit):
with self.getSession() as s: with self.getSession() as s:
logger.info(f"Updating hit! {hit.id}") logger.info(f"Updating hit! {hit.id}")
# s.flush() # s.flush()
self.triggerUpdateHooks(hit)
def saveAssignment(self, assignment): def addHIT(self, hit: HIT):
with self.getSession() as s: with self.getSession() as s:
logger.info(f"Updating assignment! {assignment.id}") s.add(hit)
# s.flush() s.flush()
self.triggerUpdateHooks(assignment.hit) s.refresh(hit)
logger.info(f"Added {hit.id}")
# def addHIT(self, hit: HIT):
# with self.getSession() as s:
# s.add(hit)
# s.flush()
# s.refresh(hit)
# logger.info(f"Added {hit.id}")
def getAvgDurationOfPreviousNHits(self, n) -> int: def getAvgDurationOfPreviousNHits(self, n) -> int:
latest_assignments = self.session.query(Assignment).\ latest_hits = self.session.query(HIT).\
filter(Assignment.created_at!=None).\ filter(HIT.submit_hit_at!=None).\
filter(Assignment.submit_page_at!=None).\ filter(HIT.accept_time!=None).\
order_by(Assignment.created_at.desc()).limit(n) order_by(HIT.submit_hit_at.desc()).limit(n)
durations = [] durations = []
for hit in latest_hits:
for assignment in latest_assignments: durations.append((hit.submit_hit_at - hit.accept_time).total_seconds())
durations.append((assignment.submit_page_at - assignment.created_at).total_seconds())
if not len(durations): if not len(durations):
return int(2.5*60) # default to 2.5 minutes return int(2.5*60)
return int(sum(durations) / len(durations)) return int(sum(durations) / len(durations))
def getEstimatedHitDuration(self): def getEstimatedHitDuration(self):
@ -357,3 +204,14 @@ class Store:
filter(HIT.submit_hit_at != None).\ filter(HIT.submit_hit_at != None).\
order_by(HIT.submit_hit_at.desc()).limit(n) order_by(HIT.submit_hit_at.desc()).limit(n)
# def rmSource(self, id: int):
# with self.getSession() as session:
# source = session.query(Source).get(id)
# if not source:
# logging.warning(f"Source nr {id} not found")
# else:
# logging.info(f"Deleting source {source.id}: {source.url}")
# session.delete(source)
#
# def getRandomNewsItem(self, session) -> NewsItem:
# return session.query(NewsItem).order_by(func.random()).limit(1).first()

254
sorteerhoed/central_management.py
View file

@ -17,14 +17,10 @@ from PIL import Image
import datetime import datetime
from shutil import copyfile from shutil import copyfile
import colorsys import colorsys
import tqdm
class Level(object): class Level(object):
""" # Level effect adapted from https://stackoverflow.com/a/3125421
Level image effect adapted from https://stackoverflow.com/a/3125421
"""
def __init__(self, minv, maxv, gamma): def __init__(self, minv, maxv, gamma):
self.minv= minv/255.0 self.minv= minv/255.0
self.maxv= maxv/255.0 self.maxv= maxv/255.0
@ -85,7 +81,6 @@ class CentralManagement():
self.isScanning = threading.Event() self.isScanning = threading.Event()
self.scanLock = threading.Lock() self.scanLock = threading.Lock()
self.notPaused = threading.Event() self.notPaused = threading.Event()
self.lightStatus = 0
def loadConfig(self, filename, args): def loadConfig(self, filename, args):
@ -99,7 +94,6 @@ class CentralManagement():
'hit_store.db' 'hit_store.db'
) )
self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO) self.store = HITStore.Store(varDb, logLevel=logging.DEBUG if self.debug else logging.INFO)
self.store.registerUpdateHook(self.updateLightHook) # change light based on status
self.logger.debug(f"Loaded configuration: {self.config}") self.logger.debug(f"Loaded configuration: {self.config}")
@ -148,10 +142,6 @@ class CentralManagement():
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
) )
self.mturk.delete_hit(HITId=pending_hit['HITId']) self.mturk.delete_hit(HITId=pending_hit['HITId'])
staleHit = self.store.getHitByRemoteId(pending_hit['HITId'])
staleHit.delete()
self.store.saveHIT(staleHit)
self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning) self.sqs = SqsListener(self.config, self.eventQueue, self.isRunning)
sqsThread = threading.Thread(target=self.sqs.start, name='sqs') sqsThread = threading.Thread(target=self.sqs.start, name='sqs')
@ -170,8 +160,9 @@ class CentralManagement():
# event listener: # event listener:
dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher') dispatcherThread = threading.Thread(target=self.eventListener, name='dispatcher')
dispatcherThread.start() dispatcherThread.start()
#
#
if self.args.autostart:
self.eventQueue.put(Signal('start', {'ding':'test'})) self.eventQueue.put(Signal('start', {'ding':'test'}))
while self.isRunning.is_set(): while self.isRunning.is_set():
@ -186,9 +177,8 @@ class CentralManagement():
self.expireCurrentHit() self.expireCurrentHit()
def expireCurrentHit(self): def expireCurrentHit(self):
if self.currentHit and not self.currentHit.isConfirmed(): if self.currentHit and self.currentHit.hit_id: # hit pending
if self.currentHit.hit_id: # hit pending at Amazon self.logger.warn(f"Delete hit: {self.currentHit.hit_id}")
self.logger.warn(f"Expire hit: {self.currentHit.hit_id}")
self.mturk.update_expiration_for_hit( self.mturk.update_expiration_for_hit(
HITId=self.currentHit.hit_id, HITId=self.currentHit.hit_id,
ExpireAt=datetime.datetime.fromisoformat('2015-01-01') ExpireAt=datetime.datetime.fromisoformat('2015-01-01')
@ -198,11 +188,6 @@ class CentralManagement():
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
if not self.currentHit.isSubmitted():
self.currentHit.delete()
self.store.saveHIT(self.currentHit)
def eventListener(self): def eventListener(self):
while self.isRunning.is_set(): while self.isRunning.is_set():
try: try:
@ -221,32 +206,27 @@ class CentralManagement():
- Plotter complete - Plotter complete
- -
""" """
#TODO: make level debug()
self.logger.info(f"SIGNAL: {signal}") self.logger.info(f"SIGNAL: {signal}")
if signal.name == 'start': if signal.name == 'start':
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'stop':
self.logger.warning("Stop request")
self.isRunning.clear()
elif signal.name == 'hit.scan': elif signal.name == 'hit.scan':
# start a scan
if signal.params['id'] != self.currentHit.id: if signal.params['id'] != self.currentHit.id:
self.logger.info(f"Hit.scan had wrong id: {signal}") self.logger.info(f"Hit.scanned had wrong id: {signal}")
continue continue
# self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning')) self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='scanning'))
elif signal.name == 'hit.scanned': elif signal.name == 'hit.scanned':
# TODO: wrap up hit & make new HIT # TODO: wrap up hit & make new HIT
if signal.params['hit_id'] != self.currentHit.id: if signal.params['id'] != self.currentHit.id:
self.logger.info(f"Hit.scanned had wrong id: {signal}") self.logger.info(f"Hit.scanned had wrong id: {signal}")
continue continue
self.currentHit.scanned_at = datetime.datetime.utcnow() self.currentHit.scanned_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit)
time_diff = datetime.datetime.now() - self.lastHitTime time_diff = datetime.datetime.now() - self.lastHitTime
to_wait = 10 - time_diff.total_seconds() to_wait = 10 - time_diff.total_seconds()
# self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan')) self.statusPageQueue.add(dict(hit_id=self.currentHit.id, state='scan'))
if to_wait > 0: if to_wait > 0:
self.logger.warn(f"Sleep until next hit: {to_wait}s") self.logger.warn(f"Sleep until next hit: {to_wait}s")
@ -257,11 +237,10 @@ class CentralManagement():
self.makeHit() self.makeHit()
self.lastHitTime = datetime.datetime.now() self.lastHitTime = datetime.datetime.now()
elif signal.name == 'hit.creating': elif signal.name == 'hit.creating':
# self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit')) self.statusPageQueue.add(dict(hit_id=signal.params['id'], transition='create_hit'))
pass
elif signal.name == 'hit.created': elif signal.name == 'hit.created':
# self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit')) self.statusPageQueue.add(dict(hit_id=signal.params['id'], remote_id=signal.params['remote_id'], state='hit'))
pass
elif signal.name == 'scan.start': elif signal.name == 'scan.start':
pass pass
elif signal.name == 'scan.finished': elif signal.name == 'scan.finished':
@ -272,53 +251,38 @@ class CentralManagement():
# Create new assignment # Create new assignment
if signal.params['hit_id'] != self.currentHit.id: if signal.params['hit_id'] != self.currentHit.id:
continue continue
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
assignment = self.store.newAssignment(self.currentHit)
assignment.assignment_id = signal.params['assignment_id']
self.store.saveAssignment(assignment)
self.statusPageQueue.add(dict(hit_id=self.currentHit.id, assignment_id=assignment.assignment_id, state='assignment'))
elif signal.name == 'assignment.info': elif signal.name == 'assignment.info':
assignment = self.currentHit.getAssignmentById(signal.params['assignment_id']) assignment = self.currentHit.getAssignmentById(signal.params['assignment_id'])
if not assignment: if not assignment:
self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}") self.logger.warning(f"assignment.info assignment.id not for current hit assignments: {signal}")
change = False
for name, value in signal.params.items(): for name, value in signal.params.items():
if name == 'ip': if name == 'ip':
assignment.turk_ip = value assignment.turk_ip = value
if name == 'location': if name == 'location':
assignment.turk_country = value assignment.turk_country = value
if name == 'os':
assignment.turk_os = value
if name == 'browser':
assignment.turk_browser = value
change = True
self.logger.debug(f'Set assignment: {name} to {value}')
if change: self.logger.debug(f'Set assignment: {name} to {value}')
self.store.saveAssignment(assignment) self.server.statusPage.set(name, value)
elif signal.name == 'server.open': elif signal.name == 'server.open':
self.currentHit.open_page_at = datetime.datetime.utcnow() self.currentHit.open_page_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
self.setLight(True) self.setLight(True)
elif signal.name == 'server.close': self.server.statusPage.set('state', self.currentHit.getStatus())
if not signal.params['abandoned']: self.server.statusPage.set('hit_opened', self.currentHit.open_page_at)
continue elif signal.name == 'server.submit':
a = self.currentHit.getLastAssignment() self.currentHit.submit_page_at = datetime.datetime.utcnow()
if a.assignment_id != signal.params['assignment_id']: self.store.saveHIT(self.currentHit)
self.logger.info(f"Close of older assignment_id: {signal}")
continue
self.logger.critical(f"Websocket closed of active assignment_id: {signal}")
a.abandoned_at = datetime.datetime.utcnow()
self.store.saveAssignment(a)
self.plotter.park()
self.setLight(False)
elif signal.name == 'assignment.submit':
a = self.currentHit.getLastAssignment()
if a.assignment_id != signal.params['assignment_id']:
self.logger.critical(f"Submit of invalid assignment_id: {signal}")
a.submit_page_at = datetime.datetime.utcnow()
self.store.saveAssignment(a)
self.plotter.park() self.plotter.park()
self.server.statusPage.set('hit_opened', self.currentHit.open_page_at)
# park always triggers a plotter.finished after being processed # park always triggers a plotter.finished after being processed
elif signal.name[:4] == 'sqs.': elif signal.name[:4] == 'sqs.':
@ -330,36 +294,48 @@ class CentralManagement():
sqsHit = self.currentHit sqsHit = self.currentHit
updateStatus = True updateStatus = True
assId = signal.params['event']['AssignmentId'] + '_' + signal.params['event']['WorkerId']
sqsAssignment = sqsHit.getAssignmentById(assId)
if not sqsAssignment:
self.logger.critical(f"Invalid assignmentId given for hit: {signal.params['event']}")
continue
if signal.name == 'sqs.AssignmentAccepted': if signal.name == 'sqs.AssignmentAccepted':
self.logger.info(f'Set status progress to accepted') self.logger.info(f'Set status progress to accepted')
sqsAssignment.accept_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsHit.accept_time = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
sqsAssignment.worker_id = signal.params['event']['WorkerId'] sqsHit.worker_id = signal.params['event']['WorkerId']
if updateStatus:
self.server.statusPage.set('worker_id', sqsHit.worker_id)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAccepted', 'EventTimestamp': '2019-10-23T20:16:10Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentAbandoned': elif signal.name == 'sqs.AssignmentAbandoned':
self.logger.info(f'Set status progress to abandoned') self.logger.info(f'Set status progress to abandoned')
#{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} #{'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentAbandoned', 'EventTimestamp': '2019-10-23T20:23:06Z', 'HITId': '3JHB4BPSFKKFQ263K4EFULI3LC79QJ', 'AssignmentId': '3U088ZLJVL450PB6MJZUIIUCB6VW0Y', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
sqsAssignment.abandoned_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsHit.accept_time = None
# TODO temporarily dissable block for workers that abandon, until we're sure it works sqsHit.open_page_at = None
if self.currentHit.id == sqsHit.id:
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.') self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Accepted task without working on it.')
elif signal.name == 'sqs.AssignmentReturned': elif signal.name == 'sqs.AssignmentReturned':
self.logger.info(f'Set status progress to returned') self.logger.info(f'Set status progress to returned')
sqsAssignment.rejected_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsHit.accept_time = None
sqsHit.open_page_at = None
if self.currentHit.id == sqsHit.id:
if not sqsHit.submit_page_at:
self.reset()
else:
sqsHit.submit_hit_at = datetime.datetime.utcnow() # fake submit
if updateStatus:
self.setLight(False)
# {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}} # {'event': {'HITGroupId': '301G7MYOAJ85NEW128ZDGF5DSBW53S', 'EventType': 'AssignmentReturned', 'EventTimestamp': '2019-10-23T20:16:47Z', 'HITId': '3IH9TRB0FBAKKZFP3JUD6D9YWQ1I1F', 'AssignmentId': '3BF51CHDTWLN3ZGHRKDUHFKPWIJ0H3', 'WorkerId': 'A1CK46PK9VEUH5', 'HITTypeId': '3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0'}}
elif signal.name == 'sqs.AssignmentSubmitted': elif signal.name == 'sqs.AssignmentSubmitted':
# {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'} # {'MessageId': '4b37dfdf-6a12-455d-a111-9a361eb54d88', 'ReceiptHandle': 'AQEBHc0yAdIrEmAV3S8TIoDCRxrItDEvjy0VQko56/Lb+ifszC0gdZ0Bbed24HGHZYr5DSnWkgBJ/H59ZXxFS1iVEH9sC8+YrmKKOTrKvW3gj6xYiBU2fBb8JRq+sEiNSxWLw2waxr1VYdpn/SWcoOJCv6PlS7P9EB/2IQ++rCklhVwV7RfARHy4J87bjk5R3+uEXUUi00INhCeunCbu642Mq4c239TFRHq3mwM6gkdydK+AP1MrXGKKAE1W5nMbwEWAwAN8KfoM1NkkUg5rTSYWmxxZMdVs/QRNcMFKVSf1bop2eCALSoG6l3Iu7+UXIl4HLh+rHp4bc8NoftbUJUii8YXeiNGU3wCM9T1kOerwYVgksK93KQrioD3ee8navYExQRXne2+TrUZUDkxRIdtPGA==', 'MD5OfBody': '01ccb1efe47a84b68704c4dc611a4d8d', 'Body': '{"Events":[{"Answer":"<?xml version=\\"1.0\\" encoding=\\"ASCII\\"?><QuestionFormAnswers xmlns=\\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd\\"><Answer><QuestionIdentifier>surveycode<\\/QuestionIdentifier><FreeText>test<\\/FreeText><\\/Answer><\\/QuestionFormAnswers>","HITGroupId":"301G7MYOAJ85NEW128ZDGF5DSBW53S","EventType":"AssignmentSubmitted","EventTimestamp":"2019-10-30T08:01:43Z","HITId":"3NSCTNUR2ZY42ZXASI4CS5YWV0S5AB","AssignmentId":"3ZAZR5XV02TTOCBR9MCLCNQV1XKCZL","WorkerId":"A1CK46PK9VEUH5","HITTypeId":"3EYXOXDEN7RX0YSMN4UMVN01AYKZJ0"}],"EventDocId":"34af4cd7f2829216f222d4b6e66f3a3ff9ad8ea6","SourceAccount":"600103077174","CustomerId":"A1CK46PK9VEUH5","EventDocVersion":"2014-08-15"}'}
self.logger.info(f'Set status progress to submitted') self.logger.info(f'Set status progress to submitted')
sqsAssignment.answer = signal.params['event']['Answer'] # TODO: validate the content of the submission by parsing signal.params['event']['Answer'] and comparing it with sqsHit.uuid
if sqsAssignment.uuid not in sqsAssignment.answer: sqsHit.answer = signal.params['event']['Answer']
self.logger.critical(f"Not a valid answer given?! {sqsAssignment.answer}") if sqsHit.uuid not in sqsHit.answer:
self.logger.critical(f"Not a valid answer given?! {sqsHit.answer}")
if not sqsAssignment.submit_page_at: if not sqsHit.submit_page_at:
# page not submitted, hit is. Nevertheless, create new hit. # page not submitted, hit is. Nevertheless, create new hit.
try: try:
self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment') self.mturk.reject_assignment(AssignmentId=signal.params['event']['AssignmentId'], RequesterFeedback='Did not do the assignment')
@ -367,8 +343,7 @@ class CentralManagement():
self.logger.exception(e) self.logger.exception(e)
self.makeHit() self.makeHit()
else: else:
sqsAssignment.confirmed_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ") sqsHit.submit_hit_at = datetime.datetime.strptime(signal.params['event']['EventTimestamp'],"%Y-%m-%dT%H:%M:%SZ")
# block de worker na succesvolle submit, om dubbele workers te voorkomen # block de worker na succesvolle submit, om dubbele workers te voorkomen
# TODO: Disabled after worker mail, use quals instead # TODO: Disabled after worker mail, use quals instead
#self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.') #self.mturk.create_worker_block(WorkerId=signal.params['event']['WorkerId'], Reason='Every worker can only work once on the taks.')
@ -377,25 +352,20 @@ class CentralManagement():
self.store.saveHIT(sqsHit) self.store.saveHIT(sqsHit)
if updateStatus:
self.logger.warning(f'update status: {sqsHit.getStatus()}')
# TODO: have HITStore/HIT take care of this by emitting a signal
# only update status if it is the currentHit
self.server.statusPage.set('state', sqsHit.getStatus())
else:
self.logger.warning('DO NOT update status')
elif signal.name == 'plotter.finished': elif signal.name == 'plotter.finished':
# is _always_ triggered after submit due to plotter.park() if self.currentHit and self.currentHit.submit_page_at:
if self.currentHit and self.currentHit.isSubmitted(): self.setLight(False)
self.currentHit.plotted_at = datetime.datetime.utcnow()
self.store.saveHIT(self.currentHit)
self.logger.info("Start scan thread")
scan = threading.Thread(target=self.scanImage, name='scan') scan = threading.Thread(target=self.scanImage, name='scan')
scan.start() scan.start()
elif signal.name == 'plotter.parked': self.server.statusPage.set('hit_submitted', self.currentHit.submit_page_at)
# should this have the code from plotter.finished? self.server.statusPage.set('state', self.currentHit.getStatus())
pass
elif signal.name == 'scan.test':
self.logger.info("Start test scan thread")
if self.currentHit:
self.logger.error("cannot scan when HITs are already running")
else:
scan = threading.Thread(target=self.scanTestImage, name='scantest')
scan.start()
else: else:
self.logger.critical(f"Unknown signal: {signal.name}") self.logger.critical(f"Unknown signal: {signal.name}")
except Exception as e: except Exception as e:
@ -407,6 +377,7 @@ class CentralManagement():
self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'})) self.eventQueue.put(Signal('hit.creating', {'id': self.currentHit.id if self.currentHit else 'start'}))
self.server.statusPage.reset()
self.reloadConfig() # reload new config values if they are set self.reloadConfig() # reload new config values if they are set
# self.notPaused.wait() # self.notPaused.wait()
@ -416,6 +387,7 @@ class CentralManagement():
self.logger.info(f"Make HIT {self.currentHit.id}") self.logger.info(f"Make HIT {self.currentHit.id}")
# question = open(self.config['amazon']['task_xml'], mode='r').read().replace("{HIT_NR}",str(self.currentHit.id))
question = '''<?xml version="1.0" encoding="UTF-8"?> question = '''<?xml version="1.0" encoding="UTF-8"?>
<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"> <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
<ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL> <ExternalURL>https://guest.rubenvandeven.com:8888/draw?id={HIT_NR}</ExternalURL>
@ -450,6 +422,7 @@ class CentralManagement():
self.currentHit.hit_id = new_hit['HIT']['HITId'] self.currentHit.hit_id = new_hit['HIT']['HITId']
self.store.saveHIT(self.currentHit) self.store.saveHIT(self.currentHit)
# TODO: have HITStore/HIT take care of this by emitting a signal
# self.server.statusPage.set('hit_id', new_hit['HIT']['HITId']) # self.server.statusPage.set('hit_id', new_hit['HIT']['HITId'])
# self.server.statusPage.set('hit_created', self.currentHit.created_at) # self.server.statusPage.set('hit_created', self.currentHit.created_at)
# self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}") # self.server.statusPage.set('fee', f"${self.currentHit.fee:.2f}")
@ -512,71 +485,11 @@ class CentralManagement():
# scan.start() # scan.start()
self.server.statusPage.clearAssignment() self.server.statusPage.clearAssignment()
def scanTestImage(self) -> str:
"""
Run scanimage on scaner and returns a string with the filename
"""
with self.scanLock:
if self.config['dummy_plotter']:
self.eventQueue.put(Signal('scan.start'))
self.logger.warning("Fake scanner for a few seconds")
for i in tqdm.tqdm(range(5)):
time.sleep(1)
self.eventQueue.put(Signal('scan.finished'))
return
cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe
'-l','25' #y axis, margin from top of the scanner, hence increasing this, moves the scanned image upwards
,'-t','22', # x axis, margin from left side scanner (seen from the outside)
'-x',str(181),
'-y',str(245)
]
self.logger.info(f"{cmd}")
filename = "/tmp/testscan.jpg"
self.eventQueue.put(Signal('scan.start'))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# opens connection to scanner, but only starts scanning when output becomes ready:
o, e = proc.communicate(80)
if e:
self.logger.critical(f"Scanner caused: {e.decode()}")
# Should this clear self.isRunning.clear() ?
try:
f = io.BytesIO(o)
img = Image.open(f)
img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM)
tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma'])
tunedImg.save(filename,quality=95)
except Exception as e:
self.logger.critical("Cannot create image from scan. Did scanner work?")
self.logger.exception(e)
copyfile('www/basic.svg', filename)
time.sleep(5) # sleep a few seconds for scanner to return to start position
self.eventQueue.put(Signal('scan.finished'))
def scanImage(self) -> str: def scanImage(self) -> str:
""" """
Run scanimage on scaner and returns a string with the filename Run scanimage on scaner and returns a string with the filename
""" """
with self.scanLock:
if self.config['dummy_plotter']:
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start'))
self.logger.warning("Fake scanner for a few seconds")
for i in tqdm.tqdm(range(5)):
time.sleep(1)
self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.finished'))
return
cmd = [ cmd = [
'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff', 'sudo', 'scanimage', '-d', 'epkowa', '--format', 'tiff',
'--resolution=100', # lower res, faster (more powerful) scan & wipe '--resolution=100', # lower res, faster (more powerful) scan & wipe
@ -588,6 +501,7 @@ class CentralManagement():
self.logger.info(f"{cmd}") self.logger.info(f"{cmd}")
filename = self.currentHit.getImagePath() filename = self.currentHit.getImagePath()
with self.scanLock:
self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id})) self.eventQueue.put(Signal('hit.scan', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.start')) self.eventQueue.put(Signal('scan.start'))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -595,31 +509,27 @@ class CentralManagement():
o, e = proc.communicate(80) o, e = proc.communicate(80)
if e: if e:
self.logger.critical(f"Scanner caused: {e.decode()}") self.logger.critical(f"Scanner caused: {e.decode()}")
# Should this clear self.isRunning.clear() ? #TODO: should clear self.isRunning.clear() ?
try: try:
f = io.BytesIO(o) f = io.BytesIO(o)
img = Image.open(f) img = Image.open(f)
img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM) img = img.transpose(Image.ROTATE_90).transpose(Image.FLIP_TOP_BOTTOM)
tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma']) tunedImg = Level.level_image(img, self.config['level']['min'], self.config['level']['max'], self.config['level']['gamma'])
tunedImg.save(filename,quality=95) tunedImg.save(filename)
except Exception as e: except Exception as e:
self.logger.critical("Cannot create image from scan. Did scanner work?") self.logger.critical("Cannot create image from scan. Did scanner work?")
self.logger.exception(e) self.logger.exception(e)
# TODO: create
copyfile('www/basic.svg', filename) copyfile('www/basic.svg', filename)
time.sleep(5) # sleep a few seconds for scanner to return to start position time.sleep(5) # sleep a few seconds for scanner to return to start position
self.eventQueue.put(Signal('hit.scanned', {'hit_id':self.currentHit.id})) self.eventQueue.put(Signal('hit.scanned', {'id':self.currentHit.id}))
self.eventQueue.put(Signal('scan.finished')) self.eventQueue.put(Signal('scan.finished'))
def setLight(self, on): def setLight(self, on):
value = 1 if on else 0 value = 1 if on else 0
if self.lightStatus == value:
return
self.lightStatus = value
cmd = [ cmd = [
'usbrelay', f'HURTM_1={value}' 'usbrelay', f'HURTM_1={value}'
] ]
@ -627,19 +537,3 @@ class CentralManagement():
code = subprocess.call(cmd) code = subprocess.call(cmd)
if code > 0: if code > 0:
self.logger.warning(f"Error on light change: {code}") self.logger.warning(f"Error on light change: {code}")
def updateLightHook(self, hit = None):
# ignore hit attribute, which comes from the HITstore
self.setLight(self.getLightStatus())
def getLightStatus(self) -> bool:
if not self.currentHit:
return False
a = self.currentHit.getLastAssignment()
if not a:
return False
if a.abandoned_at or a.rejected_at:
return False
if self.currentHit.plotted_at: # wait till plotter is done
return False
return True

49
sorteerhoed/plotter.py
View file

@ -34,36 +34,19 @@ class Plotter:
topLeft = absPlotWidth / self.plotWidth #ignore changes in config plotwidth topLeft = absPlotWidth / self.plotWidth #ignore changes in config plotwidth
self.q.put([(1/2.54)/absPlotWidth + topLeft,0,0]) self.q.put([(1/2.54)/absPlotWidth + topLeft,0,0])
def disable_motors(self):
self.ad.plot_setup()
self.setPenDown(False)
self.ad.options.mode = "manual"
self.ad.options.manual_cmd = "disable_xy"
self.ad.plot_run()
def connect(self): def start(self):
try:
if not self.config['dummy_plotter']:
self.ad = axidraw.AxiDraw()
# connect/disconnect once because often first connection attempt fails
self.ad.interactive()
self.ad.connect()
self.ad.pen_raise()
self.ad.disconnect()
# start?
self.ad.interactive() self.ad.interactive()
# self.ad.plot_path()
connected = self.ad.connect() connected = self.ad.connect()
if not connected:
raise Exception("Cannot connect to Axidraw")
# if not connected:
# raise Exception("Cannot connect to Axidraw")
while not connected:
self.logger.error("Cannot connect to Axidraw (retry, 1s)")
self.ad.disconnect()
time.sleep(1)
connected = self.ad.connect()
# back to default control mode:
self.ad.options.mode = "plot"
# self.ad.options.units = 1 # set to use centimeters instead of inches # self.ad.options.units = 1 # set to use centimeters instead of inches
self.ad.options.accel = 100; self.ad.options.accel = 100;
self.ad.options.speed_penup = 100 self.ad.options.speed_penup = 100
@ -71,24 +54,11 @@ class Plotter:
self.ad.options.model = 1 # 2 for A3, 1 for A4 self.ad.options.model = 1 # 2 for A3, 1 for A4
self.ad.options.pen_pos_up = 100 self.ad.options.pen_pos_up = 100
def reconnect(self):
self.connect()
# no park on intial connection
self.park() self.park()
# self.ad.moveto(0,0) # self.ad.moveto(0,0)
def start(self):
try:
if not self.config['dummy_plotter']:
self.ad = axidraw.AxiDraw()
self.connect()
else: else:
self.ad = None self.ad = None
while True:
self.logger.info("Fake AD-connect issue")
time.sleep(1)
self.axiDrawCueListener() self.axiDrawCueListener()
except Exception as e: except Exception as e:
self.logger.exception(e) self.logger.exception(e)
@ -170,11 +140,6 @@ class Plotter:
#if self.goPark: #if self.goPark:
# print("seg",segment) # print("seg",segment)
if segment == "disable_motors":
self.disable_motors()
elif segment == "reconnect":
self.reconnect()
else:
# change of pen state? draw previous segments! # change of pen state? draw previous segments!
if (segment[2] == 1 and not self.pen_down) or (segment[2] == 0 and self.pen_down) or len(segments) > 150: if (segment[2] == 1 and not self.pen_down) or (segment[2] == 0 and self.pen_down) or len(segments) > 150:
if len(segments): if len(segments):

363
sorteerhoed/webserver.py
View file

@ -21,14 +21,6 @@ import html
logger = logging.getLogger("sorteerhoed").getChild("webserver") logger = logging.getLogger("sorteerhoed").getChild("webserver")
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return o.isoformat(timespec='seconds')
return super().default(self, o)
class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler): class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
def set_extra_headers(self, path): def set_extra_headers(self, path):
"""For subclass to add extra headers to the response""" """For subclass to add extra headers to the response"""
@ -44,6 +36,7 @@ class StaticFileWithHeaderHandler(tornado.web.StaticFileHandler):
self.set_header("Content-Type", "image/svg+xml") self.set_header("Content-Type", "image/svg+xml")
class WebSocketHandler(tornado.websocket.WebSocketHandler): class WebSocketHandler(tornado.websocket.WebSocketHandler):
""" """
Websocket from the workers Websocket from the workers
@ -56,8 +49,6 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.plotterQ = plotterQ self.plotterQ = plotterQ
self.eventQ = eventQ self.eventQ = eventQ
self.store = store self.store = store
self.assignment_id = None
self.abandoned = False
def check_origin(self, origin): def check_origin(self, origin):
parsed_origin = urlparse(origin) parsed_origin = urlparse(origin)
@ -75,19 +66,12 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.hit = self.store.currentHit self.hit = self.store.currentHit
# my core assumption about assignment_id was wrong. It is not unique per worker, so we need to merge those
self.assignment_id = str(self.get_query_argument('assignmentId'))
self.assignment_id += '_' + str(self.get_query_argument('workerId'))
self.assignment = self.hit.getLastAssignment() self.assignment_id = int(self.get_query_argument('assignment_id'))
if self.assignment.assignment_id != self.assignment_id: self.timeout = datetime.datetime.now() + datetime.timedelta(seconds=self.store.getHitTimeout())
raise Exception(f"Opening websocket for invalid assignment {self.assignment_id}")
self.timeout = self.assignment.created_at + datetime.timedelta(seconds=self.store.getHitTimeout()) if self.hit.submit_hit_at:
# timeLeft = (self.timeout - datetime.datetime.utcnow()).total_seconds()
if self.hit.isSubmitted():
raise Exception("Opening websocket for already submitted hit") raise Exception("Opening websocket for already submitted hit")
#logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}") #logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}")
@ -95,22 +79,21 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.strokes = [] self.strokes = []
# the client sent the message # the client sent the message
def on_message(self, message): def on_message(self, message):
logger.debug(f"recieve: {message}") logger.debug(f"recieve: {message}")
if self.assignment_id != self.hit.getLastAssignment().assignment_id: if self.assignment_id != self.hit.getLastAssignment().assignment_id:
logger.critical(f"Skip message for non-last assignment {message}") logger.critical(f"Skip message for non-last assignment {message}")
return
if datetime.datetime.utcnow() > self.timeout: if datetime.datetime.now() > self.timeout:
logger.critical("Close websocket after timeout (abandon?)") logger.critical("Close websocket after timeout (abandon?)")
self.close() self.close()
return return
try: try:
msg = json.loads(message) msg = json.loads(message)
# TODO: sanitize input: min/max, limit strokes
if msg['action'] == 'move': if msg['action'] == 'move':
# TODO: min/max input # TODO: min/max input
point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])] point = [float(msg['direction'][0]),float(msg['direction'][1]), bool(msg['mouse'])]
@ -144,8 +127,8 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
self.write_message(json.dumps({ self.write_message(json.dumps({
'action': 'submitted', 'action': 'submitted',
'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.assignment.uuid}", 'msg': f"Submission ok, please copy this token to your HIT at Mechanical Turk: {self.hit.uuid}",
'code': str(self.assignment.uuid) 'code': str(self.hit.uuid)
})) }))
self.close() self.close()
@ -153,9 +136,8 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
# not used, implicit in move? # not used, implicit in move?
pass pass
elif msg['action'] == 'info': elif msg['action'] == 'info':
self.eventQ.put(Signal('assignment.info', dict( self.eventQ.put(Signal('hit.info', dict(
hit_id=self.hit.id, hit_id=self.hit.id,
assignment_id=self.assignment_id,
resolution=msg['resolution'], resolution=msg['resolution'],
browser=msg['browser'] browser=msg['browser']
))) )))
@ -171,45 +153,38 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
# client disconnected # client disconnected
def on_close(self): def on_close(self):
self.__class__.rmConnection(self) self.__class__.rmConnection(self)
if self.assignment_id:
self.eventQ.put(Signal('server.close', dict(assignment_id=self.assignment_id, abandoned=self.abandoned)))
logger.info(f"Client disconnected: {self.request.remote_ip}") logger.info(f"Client disconnected: {self.request.remote_ip}")
# TODO: abandon assignment??
def submit_strokes(self): def submit_strokes(self):
if len(self.strokes) < 1: if len(self.strokes) < 1:
return False return False
self.eventQ.put(Signal("assignment.submit", dict( self.eventQ.put(Signal("server.submit", dict(hit_id = self.hit.id)))
hit_id = self.hit.id,
assignment_id=self.assignment_id)))
# deprecated: now done at scanner method: if self.config['dummy_plotter']:
# if self.config['dummy_plotter']: d = strokes2D(self.strokes)
# d = strokes2D(self.strokes) svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?>
# svg = f"""<?xml version="1.0" encoding="UTF-8" standalone="no"?> <svg viewBox="0 0 600 600"
# <svg viewBox="0 0 600 600" xmlns:dc="http://purl.org/dc/elements/1.1/"
# xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#"
# xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
# xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:svg="http://www.w3.org/2000/svg"
# xmlns:svg="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg"
# xmlns="http://www.w3.org/2000/svg" version="1.1"
# version="1.1" >
# > <path d="{d}" style="stroke:black;stroke-width:2;fill:none;" />
# <path d="{d}" style="stroke:black;stroke-width:2;fill:none;" /> </svg>
# </svg> """
# """
# filename = self.hit.getImagePath()
# filename = self.hit.getImagePath() logger.info(f"Write to {filename}")
# logger.info(f"Write to {filename}") with open(filename, 'w') as fp:
# with open(filename, 'w') as fp: fp.write(svg)
# fp.write(svg)
# we fake a hit.scanned event # we fake a hit.scanned event
# self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id})) self.eventQ.put(Signal('hit.scanned', {'hit_id':self.hit.id}))
return self.assignment.uuid return self.hit.uuid
@classmethod @classmethod
def rmConnection(cls, client): def rmConnection(cls, client):
@ -217,109 +192,15 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
return return
cls.connections.remove(client) cls.connections.remove(client)
@classmethod
def hasConnection(cls, client):
return client in cls.connections
@classmethod
def timeoutConnectionForAssignment(cls, assignment_id):
logger.warn(f"Check timeout for {assignment_id}")
for client in cls.connections:
logger.info(client.assignment_id)
if client.assignment_id == assignment_id:
client.abandoned = True
client.close()
class ConfigWebSocketHandler(tornado.websocket.WebSocketHandler):
"""
Websocket for config & control
"""
CORS_ORIGINS = ['localhost', '192.168.1.102','guest.rubenvandeven.com']
connections = set()
def initialize(self, config, plotterQ: Queue, eventQ: Queue, store: HITStore):
self.config = config
self.plotterQ = plotterQ
self.eventQ = eventQ
self.store = store
def check_origin(self, origin):
parsed_origin = urlparse(origin)
# parsed_origin.netloc.lower() gives localhost:3333
valid = any([parsed_origin.hostname.endswith(origin) for origin in self.CORS_ORIGINS])
logger.info(f"Connection from {origin} is valid? valid: {valid}")
return valid
# the client connected
def open(self, p = None):
self.__class__.connections.add(self)
logger.warning(f"Config client connected: {self.request.remote_ip}")
#logger.info(f"New client connected: {self.request.remote_ip} for {self.hit.id}/{self.hit.hit_id}")
# self.eventQ.put(Signal('server.open', dict(assignment_id=self.assignment_id)))
# self.strokes = []
# the client sent the message
def on_message(self, message):
logger.debug(f"recieve: {message}")
try:
msg = json.loads(message)
if msg['action'] == 'start':
self.eventQ.put(Signal('start', {'ding':'test'}))
elif msg['action'] == 'disable_motors':
self.plotterQ.put("disable_motors")
elif msg['action'] == 'enable_motors':
self.plotterQ.put("reconnect")
elif msg['action'] == 'scanner_test':
self.eventQ.put(Signal('scan.test'))
elif msg['action'] == 'stop':
self.eventQ.put(Signal('stop'))
# elif msg['action'] == 'info':
# self.eventQ.put(Signal('assignment.info', dict(
# hit_id=self.hit.id,
# assignment_id=self.assignment_id,
# resolution=msg['resolution'],
# browser=msg['browser']
# )))
# pass
else:
# self.send({'alert': 'Unknown request: {}'.format(message)})
logger.warn('Unknown request: {}'.format(message))
except Exception as e:
# self.send({'alert': 'Invalid request: {}'.format(e)})
logger.exception(e)
# client disconnected
def on_close(self):
self.__class__.rmConnection(self)
logger.warning(f"Config client disconnected: {self.request.remote_ip}")
# TODO: abandon assignment??
@classmethod
def rmConnection(cls, client):
if client not in cls.connections:
return
cls.connections.remove(client)
@classmethod
def hasConnection(cls, client):
return client in cls.connections
class StatusWebSocketHandler(tornado.websocket.WebSocketHandler): class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
CORS_ORIGINS = ['localhost'] CORS_ORIGINS = ['localhost']
connections = set() connections = set()
queue = queue.Queue()
def initialize(self, statusPage): def initialize(self, statusPage):
self.statusPage = statusPage self.statusPage = statusPage
pass
def check_origin(self, origin): def check_origin(self, origin):
parsed_origin = urlparse(origin) parsed_origin = urlparse(origin)
@ -328,12 +209,14 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
return valid return valid
# the client connected # the client connected
def open(self): def open(self, p = None):
self.__class__.connections.add(self) self.__class__.connections.add(self)
limit = 2 for prop, value in self.statusPage.__dict__.items():
if 'all' in self.request.query_arguments: self.write_message(json.dumps({
limit = None 'property': prop,
self.write_message(json.dumps(self.statusPage.fetch(limit), cls=DateTimeEncoder)) 'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value
}))
# client disconnected # client disconnected
def on_close(self): def on_close(self):
@ -347,13 +230,13 @@ class StatusWebSocketHandler(tornado.websocket.WebSocketHandler):
cls.connections.remove(client) cls.connections.remove(client)
@classmethod @classmethod
def update_for_all(cls, data): def update_for_all(cls, prop, value):
logger.debug(f"update for all {data}") logger.debug(f"update for all {prop} {value}")
for connection in cls.connections: for connection in cls.connections:
try: connection.write_message(json.dumps({
connection.write_message(json.dumps(data, cls=DateTimeEncoder)) 'property': prop,
except Exception as e: 'value': value.isoformat(timespec='seconds') if type(value) is datetime.datetime else value
logger.exception(e) }))
def strokes2D(strokes): def strokes2D(strokes):
# strokes to a d attribute for a path # strokes to a d attribute for a path
@ -394,38 +277,17 @@ class DrawPageHandler(tornado.web.RequestHandler):
try: try:
hit_id = int(self.get_query_argument('id')) hit_id = int(self.get_query_argument('id'))
if hit_id != self.store.currentHit.id: if hit_id != self.store.currentHit.id:
assignmentId = self.get_query_argument('assignmentId', '')
orig_assigmentId = assignmentId
if len(assignmentId):
assignmentId += '_' + str(self.get_query_argument('workerId', ''))
hit = self.store.getHitById(hit_id)
assignment = hit.getAssignmentById(assignmentId)
if not assignment:
self.write("Invalid HIT or assignment id")
return
submitUrl = self.get_query_argument('turkSubmitTo', '')
submitUrl += '/mturk/externalSubmit'
self.write("An error occured. Please re-submit your assignment validation code. We're really sorry for the inconvenience.")
self.write(f"<form method='post' action='{submitUrl}'>")
self.write(f"<input type='text' name='assignmentId' value='{orig_assigmentId}'>")
self.write(f"<input type='text' name='surveycode' value='{assignment.uuid}'>")
self.write(f"<input type='submit' value='Submit finished assignment'>")
self.write("</form>")
self.write("Invalid HIT") self.write("Invalid HIT")
return return
hit = self.store.currentHit hit = self.store.currentHit
except Exception: except Exception:
self.write("HIT not found") self.write("HIT not found")
else: else:
if hit.isSubmitted(): if hit.submit_page_at:
self.write("HIT already submitted") self.write("HIT already submitted")
return return
assignmentId = self.get_query_argument('assignmentId', '') assignmentId = self.get_query_argument('assignmentId', '')
if len(assignmentId) and assignmentId != "ASSIGNMENT_ID_NOT_AVAILABLE":
assignmentId += '_' + str(self.get_query_argument('workerId', ''))
if len(assignmentId) < 1: if len(assignmentId) < 1:
logger.critical("Accessing page without assignment id. Allowing it for debug purposes... fingers crossed?") logger.critical("Accessing page without assignment id. Allowing it for debug purposes... fingers crossed?")
@ -433,18 +295,6 @@ class DrawPageHandler(tornado.web.RequestHandler):
if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE': if assignmentId == 'ASSIGNMENT_ID_NOT_AVAILABLE':
previewOnly = True previewOnly = True
if len(assignmentId) and not previewOnly:
# process/create assignment
assignment = self.store.currentHit.getAssignmentById(assignmentId)
if not assignment:
# new assignment
logger.warning(f"Create new assignment {assignmentId}")
assignment = self.store.newAssignment(self.store.currentHit, assignmentId)
assignment.worker_id = str(self.get_query_argument('workerId', ''))
self.store.saveAssignment(assignment)
logger.info(f"Set close timeout for {self.store.getHitTimeout()}")
Server.loop.asyncio_loop.call_later(self.store.getHitTimeout(), WebSocketHandler.timeoutConnectionForAssignment, assignment.assignment_id)
previous_hit = self.store.getLastSubmittedHit() previous_hit = self.store.getLastSubmittedHit()
if not previous_hit: if not previous_hit:
# start with basic svg # start with basic svg
@ -464,7 +314,7 @@ class DrawPageHandler(tornado.web.RequestHandler):
.replace("{TOP_PADDING}", str(self.top_padding))\ .replace("{TOP_PADDING}", str(self.top_padding))\
.replace("{LEFT_PADDING}", str(self.left_padding))\ .replace("{LEFT_PADDING}", str(self.left_padding))\
.replace("{SCRIPT}", '' if previewOnly else '<script type="text/javascript" src="/assignment.js"></script>')\ .replace("{SCRIPT}", '' if previewOnly else '<script type="text/javascript" src="/assignment.js"></script>')\
.replace("{ASSIGNMENT}", '' if previewOnly else str(assignment.getOriginalAssignmentId())) # TODO: fix unsafe inserting of GET variable .replace("{ASSIGNMENT}", '' if previewOnly else str(assignmentId)) # TODO: fix unsafe inserting of GET variable
self.write(contents) self.write(contents)
@ -480,8 +330,7 @@ class DrawPageHandler(tornado.web.RequestHandler):
self.eventQ.put(Signal('hit.assignment', dict( self.eventQ.put(Signal('hit.assignment', dict(
hit_id=hit.id, ip=ip, assignment_id=assignmentId hit_id=hit.id, ip=ip, assignment_id=assignmentId
))) )))
# self.eventQ.put(Signal('hit.info', dict(hit_id=hit.id, ip=ip)))
self.eventQ.put(Signal('assignment.info', dict(assignment_id=assignmentId, ip=ip)))
try: try:
geoip = self.geoip_reader.country(ip) geoip = self.geoip_reader.country(ip)
@ -504,30 +353,30 @@ class BackendHandler(tornado.web.RequestHandler):
def get(self): def get(self):
rows = [] rows = []
# for hit in self.store.getHITs(100): for hit in self.store.getHITs(100):
# if hit.submit_hit_at and hit.accept_time: if hit.submit_hit_at and hit.accept_time:
# seconds = (hit.submit_hit_at - hit.accept_time).total_seconds() seconds = (hit.submit_hit_at - hit.accept_time).total_seconds()
# duration_m = int(seconds/60) duration_m = int(seconds/60)
# duration_s = max(int(seconds%60), 0) duration_s = max(int(seconds%60), 0)
# duration = (f"{duration_m}m" if duration_m else "") + f"{duration_s:02d}s" duration = (f"{duration_m}m" if duration_m else "") + f"{duration_s:02d}s"
# else: else:
# duration = "-" duration = "-"
#
# fee = f"${hit.fee:.2}" if hit.fee else "-"
#
# rows.append(
# f"""
# <tr><td></td><td>{hit.worker_id}</td>
# <td>{hit.turk_ip}</td>
# <td>{hit.turk_country}</td>
# <td>{fee}</td>
# <td>{hit.accept_time}</td>
# <td>{duration}</td><td></td>
# """
# )
contents = open(os.path.join(self.path, 'backend/backend.html'), 'r').read() fee = f"${hit.fee:.2}" if hit.fee else "-"
# contents = contents.replace("{{TBODY}}", "".join(rows))
rows.append(
f"""
<tr><td></td><td>{hit.worker_id}</td>
<td>{hit.turk_ip}</td>
<td>{hit.turk_country}</td>
<td>{fee}</td>
<td>{hit.accept_time}</td>
<td>{duration}</td><td></td>
"""
)
contents = open(os.path.join(self.path, 'backend.html'), 'r').read()
contents = contents.replace("{{TBODY}}", "".join(rows))
self.write(contents) self.write(contents)
class StatusPage(): class StatusPage():
@ -535,40 +384,58 @@ class StatusPage():
Properties for on the status page, which are send over websockets the moment Properties for on the status page, which are send over websockets the moment
they are altered. they are altered.
""" """
def __init__(self):
self.reset()
def __init__(self, store: HITStore): def reset(self):
self.store = store logger.info("Resetting status")
self.store.registerUpdateHook(self) self.hit_id = None
self.worker_id = None
self.ip = None
self.location = None
self.browser = None
self.os = None
self.resolution = None
self.state = None
self.fee = None
self.hit_created = None
self.hit_opened = None
self.hit_submitted = None
def update(self, hit = None): def clearAssignment(self):
""" logger.info("Resetting hit assignment")
Send the given HIT formatted to the websocket clients self.worker_id = None
self.ip = None
self.location = None
self.browser = None
self.os = None
self.resolution = None
self.hit_created = None
If no hit is given, load the last 2 items def __setattr__(self, name, value):
""" if name in self.__dict__ and self.__dict__[name] == value:
if hit: logger.debug(f"Ignore setting status of {name}: it already is set to {value}")
data = [hit.toDict()] return
else:
hits = self.store.getNewestHits(2)
data = [hit.toDict() for hit in hits]
self.__dict__[name] =value
logger.info(f"Update status: {name}: {value}")
if Server.loop: if Server.loop:
Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, data) Server.loop.asyncio_loop.call_soon_threadsafe(StatusWebSocketHandler.update_for_all, name, value)
else: else:
logger.warn("Status: no server loop to call update command") logger.warn("Status: no server loop to call update command")
def fetch(self, limit = 2):
""" def set(self, name, value):
Fetch latest, used on connection of status page return self.__setattr__(name, value)
"""
hits = self.store.getNewestHits(limit)
return [hit.toDict() for hit in hits]
class Server: class Server:
""" """
Server for HIT -> plotter events Server for HIT -> plotter events
As well as for the Status interface As well as for the Status interface
TODO: change to have the HIT_id as param to the page. Load hit from storage with previous image
""" """
loop = None loop = None
@ -585,7 +452,7 @@ class Server:
self.server_loop = None self.server_loop = None
self.store = store self.store = store
self.statusPage = StatusPage(store) self.statusPage = StatusPage()
def start(self): def start(self):
@ -603,12 +470,6 @@ class Server:
'eventQ': self.eventQ, 'eventQ': self.eventQ,
'store': self.store, 'store': self.store,
}), }),
(r"/config/ws(.*)", ConfigWebSocketHandler, {
'config': self.config,
'plotterQ': self.plotterQ,
'eventQ': self.eventQ,
'store': self.store,
}),
(r"/status/ws", StatusWebSocketHandler, dict(statusPage = self.statusPage)), (r"/status/ws", StatusWebSocketHandler, dict(statusPage = self.statusPage)),
(r"/draw", DrawPageHandler, (r"/draw", DrawPageHandler,
dict( dict(
@ -628,8 +489,6 @@ class Server:
store = self.store, store = self.store,
path=self.web_root, path=self.web_root,
)), )),
(r"/frames/(.*)", StaticFileWithHeaderHandler,
{"path": 'scanimation/interfaces/frames'}),
(r"/(.*)", StaticFileWithHeaderHandler, (r"/(.*)", StaticFileWithHeaderHandler,
{"path": self.web_root}), {"path": self.web_root}),
], debug=True, autoreload=False) ], debug=True, autoreload=False)

2
www/assignment.js
View file

@ -51,7 +51,7 @@ let draw = function(e) {
strokeEl.setAttribute('d', d); strokeEl.setAttribute('d', d);
} }
//console.log([pos['x'], pos['y']], isDrawing); console.log([pos['x'], pos['y']], isDrawing);
socket.send(JSON.stringify({ socket.send(JSON.stringify({
'action': 'move', 'action': 'move',
'direction': [pos['x'], pos['y']], 'direction': [pos['x'], pos['y']],

40
www/backend/backend.html
View file

@ -1,40 +0,0 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="stylesheet" type="text/css" href="/backend/style.css" />
<script src='../worker_specs/dateformat.js'></script>
<script src='../worker_specs/reconnecting-websocket.min.js'></script>
<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
</head>
<body>
<div id="wrapper">
<div v-if="hit && hit.assignment" :class="[{'invisible': !hit.visible}, 'hit']">
<img :src="hit.scan_image">
<div class='credits'>
<span class='worker_id'>{{hit.assignment.worker_id}}</span>
<span class='country'>{{hit.assignment.turk_country}}</span>
<template v-if="hit.preceding_assignments.length">
<div id='collaborators'>
<div id='collab_items'>
<span v-for="a of hit.preceding_assignments" class='credit'>
<span class='worker_id'>{{a.worker_id}}</span>
<span class='country'>{{a.turk_country_code}}</span>
</span>
</div>
</div>
</template>
</div>
</div>
</div>
<script src="/backend/script.js"></script>
</body>
</html>

60
www/backend/script.js
View file

@ -1,60 +0,0 @@
var app = new Vue({
el: '#wrapper',
data: {
hit: {}
},
})
var hits = {};
var hitIds = [];
// fetch ?all in database
let ws = new ReconnectingWebSocket('ws://localhost:8888/status/ws?all')
ws.addEventListener('open', () => {
// ws.send('hi server')
})
ws.addEventListener('message', (event) => {
console.log('message: ')
let load_hits = JSON.parse(event.data)
for(let hit of load_hits) {
console.log(hit);
hits[hit.id] = hit;
if(hit.scanned_at && hitIds.indexOf(hit.id) < 0){
// only add if indeed scanned
hitIds.push(hit.id);
}
}
})
var currentI = 0;
setInterval(function(e){
if(hitIds.length < 1) {
return;
}
currentI = (currentI + 1) % hitIds.length;
let currentHitId = hitIds[currentI];
console.log(currentHitId);
let hit = hits[currentHitId];
hit['visible'] = false;
app.hit['visible'] = false;
setTimeout(() => {
app.hit = hits[currentHitId]
}, 1000);
setTimeout(() => {
app.hit['visible'] = true;
let wrapperEl = document.getElementById("collaborators");
let innerEl = document.getElementById("collab_items");
let size = 80;
do {
innerEl.style.fontSize = size + "%";
size --;
} while(innerEl.clientHeight > wrapperEl.clientHeight && size > 27);
}, 1100);
}, 7000);

129
www/backend/style.css
View file

@ -1,129 +0,0 @@
@font-face {
font-family: 'bebas';
src: url('/font/BebasNeue-Regular.ttf');
}
@font-face {
font-family: 'freesans';
src: url('/font/FreeSans.ttf')
}
@font-face {
font-family: 'bt';
src: url('/font/steelfish_rg.ttf')
}
:root{
--base-font-size: 60px;
--spec_name-font-size: 120%;
--spec_value-font-size: 250%;
--base-color: #271601; /* tekst */
--alt-color: #FFF5DF; /* achtergrond */
--amazon-color: #F0C14C;
/* ////// GAT ACHTERKANT PLEK & POSITIE /////// */
/* */ /* */
/* */ --pos-x: 0px; /* */
/* */ --pos-y: 50px; /* */
/* */ --width: 100%; /* 285mm */
/* */ --height: 100%; /* 500mm */
/* */ /* */
/* //////////////////////////////////////////// */
}
body{
background: black;
margin:0;
color:#f5f5f5;
font-family: 'bt';
font-size: var(--base-font-size);
line-height: var(--base-font-size)*1.5;
}
.hit{
position:absolute;
top:var(--pos-y);
left:24px;
right:21px;
bottom:0;
text-align: center;
transition: opacity 1s;
opacity: 1;
/*animation: fadeIn 1s, fadeOut 1s 2s;*/
}
.hit.invisible{
opacity: 0;
}
@keyframes fadeIn{
from{opacity:0;}
to{opacity:1;}
}
@keyframes fadeOut{
from{opacity:1;}
to{opacity:0;}
}
.hit img{
display: block;
margin: 6vh auto 7vh;
width: 875px;
}
.credits{
font-size: var(--base-font-size);
text-transform: uppercase;
}
.credits::before{
content:'created by';
font-family: 'freesans';
text-transform: lowercase;
display:block;
font-size: calc(var(--base-font-size)/3);
}
.country{
/* color: #fff; */
/* font-family: 'freesans';
text-transform: lowercase; */
font-size: 50%;
vertical-align: 67%;
margin-left: -5px;
margin-right: .4em;
}
.country::before{content:'(';}
.country::after{content:')';}
#collaborators{
height: 27vh;
width: 800px;
margin: 0 auto;
}
#collab_items{
text-align: justify-center;
}
#collaborators::before{
margin-top: 7vh;
content:'in collaboration with';
display:block;
font-family: 'freesans';
font-size: calc(var(--base-font-size)/3);
text-transform: lowercase;
}
#collaborators .credit{
margin-right:.1em;
white-space: nowrap;
display: inline-block;
}

23
www/basic.svg
View file

@ -5,11 +5,11 @@
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg" xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg" xmlns="http://www.w3.org/2000/svg"
width="210mm" id="svg8"
height="210mm"
viewBox="0 0 210 210"
version="1.1" version="1.1"
id="svg8"> viewBox="0 0 210 210"
height="210mm"
width="210mm">
<defs <defs
id="defs2" /> id="defs2" />
<metadata <metadata
@ -25,15 +25,14 @@
</rdf:RDF> </rdf:RDF>
</metadata> </metadata>
<g <g
style="stroke-width:1.52441633" transform="translate(0,-87)"
id="layer1" id="layer1">
transform="matrix(0.65598879,0,0,0.65598879,44.11553,-86.509667)">
<rect <rect
style="fill:none;fill-opacity:1;stroke:#ffffff;stroke-width:1.52441633;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" y="138.32738"
id="rect815"
width="107.34524"
height="107.34524"
x="51.327381" x="51.327381"
y="138.32738" /> height="107.34524"
width="107.34524"
id="rect815"
style="fill:none;fill-opacity:1;stroke:white;stroke-width:1;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" />
</g> </g>
</svg> </svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1 KiB

17
www/basic_square.svg
View file

@ -13,7 +13,7 @@
height="180mm" height="180mm"
preserveAspectRatio="none" preserveAspectRatio="none"
id="svg3" id="svg3"
sodipodi:docname="basic_square.svg" sodipodi:docname="000139.svg"
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"> inkscape:version="0.92.4 (5da689c313, 2019-01-14)">
<metadata <metadata
id="metadata9"> id="metadata9">
@ -37,23 +37,22 @@
guidetolerance="10" guidetolerance="10"
inkscape:pageopacity="0" inkscape:pageopacity="0"
inkscape:pageshadow="2" inkscape:pageshadow="2"
inkscape:window-width="3840" inkscape:window-width="3836"
inkscape:window-height="2160" inkscape:window-height="2126"
id="namedview5" id="namedview5"
showgrid="false" showgrid="false"
inkscape:zoom="1.3875926" inkscape:zoom="1.3875926"
inkscape:cx="-14.783892" inkscape:cx="311.32047"
inkscape:cy="589.76197" inkscape:cy="589.76197"
inkscape:window-x="0" inkscape:window-x="2"
inkscape:window-y="0" inkscape:window-y="32"
inkscape:window-maximized="1" inkscape:window-maximized="1"
inkscape:current-layer="svg3" inkscape:current-layer="svg3" />
showguides="false" />
<rect <rect
style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:4;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1" style="fill:none;fill-opacity:1;stroke:#808080;stroke-width:4;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0.75590557;stroke-opacity:1"
id="rect819" id="rect819"
width="306.99152" width="306.99152"
height="306.99152" height="306.99152"
x="1287.0762" x="1287.0762"
y="165.88982" /> y="638.77118" />
</svg> </svg>

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 1.7 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

113
www/config/control.html
View file

@ -1,113 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>GW - Control</title>
<style>
:root{
font-family: sans-serif;
background-color: black;
color: white;
font-size: 30px;
}
main{
width: 700px;
margin: 50px auto;
}
a{
background-color:lightblue;
color:white;
text-decoration: none;
cursor: pointer;
padding: 5px;
border-radius: 5px;
}
a:hover{
background-color: blue;
}
a:active{
background-color: red;;
}
li{
margin-top: 50px;
}
</style>
<script src="../worker_specs/reconnecting-websocket.min.js"></script>
<script>
let ws = new ReconnectingWebSocket('ws://'+location.hostname+':'+location.port+'/config/ws')
ws.addEventListener('open', () => {
// ws.send('hi server')
})
ws.addEventListener('message', (event) => {
console.log('message: ' + event.data)
let hits = JSON.parse(event.data)
let a = {};
for(let hitid in app.hits) {
a[hitid] = app.hits[hitid];
}
for(let hit of hits){
a[hit.id] = hit;
}
app.hits = a;
})
function disablemotors(el) {
ws.send(JSON.stringify({
'action': 'disable_motors',
}));
}
function resetPenPosition() {
ws.send(JSON.stringify({
'action': 'enable_motors',
}));
}
function testScanner() {
ws.send(JSON.stringify({
'action': 'scanner_test',
}));
}
function startServer() {
ws.send(JSON.stringify({
'action': 'start',
}));
}
function stopServer() {
ws.send(JSON.stringify({
'action': 'stop',
}));
}
</script>
</head>
<body>
<main>
<h1>Guest Worker Control Panel</h1>
<ol>
<li><a onclick="disablemotors(this)">Disable motors</a> </li>
<li>Place pen in the holder and move it to position 0,0 (bottom left, as seen from the outside)</li>
<li><a onclick="resetPenPosition()">Pen to start position</a> [pen should now move to top right position]</li>
<li>Power on scanner, verify cog position</li>
<li><a onclick="testScanner()">Test scanner</a> - if it blinks orange: turn it off by holding power; turn on again and retry</li>
<li><a onclick="startServer()">Start the work!</a></li>
<li><a href="/backend">Exit setup</a></li>
</ol>
<br>
<br>
<hr>
<br>
<br>
<h3>Stop</h3>
<ol>
<li><a onclick="stopServer()">Stop the work</a> ... then wait a few minutes before powering off computer</li>
</ol>
</main>
</body>
</html>

BIN
www/font/built titling bd.ttf
View file

Binary file not shown.

BIN
www/font/built titling el.ttf
View file

Binary file not shown.

BIN
www/font/built titling sb.ttf
View file

Binary file not shown.

BIN
www/font/built_titling_lt.ttf
View file

Binary file not shown.

BIN
www/font/built_titling_rg.ttf
View file

Binary file not shown.

BIN
www/font/steelfish bd it.ttf
View file

Binary file not shown.

BIN
www/font/steelfish bd.ttf
View file

Binary file not shown.

BIN
www/font/steelfish eb it.ttf
View file

Binary file not shown.

BIN
www/font/steelfish eb.ttf
View file

Binary file not shown.

BIN
www/font/steelfish outline.ttf
View file

Binary file not shown.

BIN
www/font/steelfish rg it.ttf
View file

Binary file not shown.

BIN
www/font/steelfish_rg.ttf
View file

Binary file not shown.

6
www/index.html
View file

@ -55,7 +55,7 @@
bottom:0; bottom:0;
left:0; left:0;
background:none; background:none;
cursor: url('/cursor.png') 6 6, auto; cursor: url(cursor.png) 6 6, auto;
} }
.gray{ .gray{
position:absolute; position:absolute;
@ -145,8 +145,8 @@
</div> </div>
<div id='info'> <div id='info'>
<ul> <ul>
<li>Drag the mouse to trace over the shape above.</li> <li>Drag the mouse to trace the lines above.</li>
<li>Follow the lines as precise as possible: there's only one image for this HIT.</li> <li>Try to be as precise as possible: there's only one image per HIT.</li>
<li>Press submit when you're done.</li> <li>Press submit when you're done.</li>
<li><strong>Please watch the clock!</strong> timing is strict because the tracing is live streamed to us. Unfortunately, due to high abandonment rates we have to keep the timer strict.</li> <li><strong>Please watch the clock!</strong> timing is strict because the tracing is live streamed to us. Unfortunately, due to high abandonment rates we have to keep the timer strict.</li>
</ul> </ul>

BIN
www/worker_specs/fake_scan.jpg (Stored with Git LFS)
View file

Binary file not shown.

113
www/worker_specs/index.html
View file

@ -5,114 +5,10 @@
<link rel="stylesheet" type="text/css" href="style.css" /> <link rel="stylesheet" type="text/css" href="style.css" />
<script src='dateformat.js'></script> <script src='dateformat.js'></script>
<script src='reconnecting-websocket.min.js'></script> <script src='reconnecting-websocket.min.js'></script>
<script src="https://cdn.jsdelivr.net/npm/vue/dist/vue.js"></script>
</head> </head>
<body> <body>
<div id="wrapper"> <div id="wrapper">
<div class='hit' v-for="(hit, hitId) in hits"
:class="[{'hugvey--on': hit.status != 'off'},'hugvey--' + hit.status]"
>
<div class='transition'>
<p v-if="hit.hit_id">Created subsequent HIT</p>
<p v-else>Creating subsequent HIT</p>
</div>
<div class='state' v-if="hit.hit_id">
<h2>human intelligence task</h2>
<p class="descriptor">task id</p>
<p>{{hit.hit_id}}</p>
<p class="descriptor">task description</p>
<p>Use the mouse to trace the image above</p>
<div class='columns'>
<div class='column'>
<p class="descriptor">fee</p>
<p>$ {{formatPrice(hit.fee)}}</p>
</div><div class='column'>
<p class="descriptor">amazon markup</p>
<p>$ {{formatPrice(hit.fee*.2)}}</p>
</div>
</div>
</div>
<div class='transition' v-if="hit.hit_id" :class="[{'no_arrow': !hit.assignment}]">
<p v-if="!hit.assignment">waiting for worker</p>
<template v-else>
<p v-if="hit.assignment.rejected_at || hit.assignment.abandoned_at">task abandoned</p>
<p v-else>task accepted</p>
</template>
</div>
<template v-if="hit.assignment">
<div class='state' v-if="hit.assignment"
:class="[{'assignment-hidden': hit.assignment.rejected_at || hit.assignment.abandoned_at}]">
<h2>guest worker</h2>
<!-- IF statement of worker id al beschikbaar is -->
<p class="descriptor">worker id</p>
<p>{{hit.assignment.worker_id}}</p>
<p class="descriptor">visiting from</p>
<!-- {{hit.assignment.turk_browser}} / / {{hit.assignment.turk_os}} ({{hit.assignment.turk_ip}}) -->
<p>{{hit.assignment.turk_country}}</p>
<p class="descriptor">system</p>
<p>{{hit.assignment.turk_browser}} - {{hit.assignment.turk_os}}</p>
</div>
<div class='transition' v-if="hit.assignment.submit_page_at">
<p v-if="!hit.assignment.confirmed_at">Assignment submitted</p>
<p v-else>Confirmed submission</p> <!-- Validated submitted code through SQSmessage -->
</div>
<div class='state' v-if="hit.assignment.submit_page_at">
<h2>worker input</h2>
<div class="svgcrop" style="height: 110px; overflow: hidden">
<img :src="hit.svg_image" style="margin-top: 0px">
</div>
<div class='columns'>
<div class='column'>
<p class="descriptor">duration</p>
<p>{{duration(hit.assignment.submit_page_at, hit.assignment.created_at)}}</p>
</div>
<div class='column'v-if="hit.path_length">
<p class="descriptor">drawing distance</p>
<p>{{hit.path_length}} pixels</p>
</div>
</div>
</div>
<div class='transition' v-if="hit.assignment.submit_page_at">
<p v-if="!hit.scanned_at">Scanning</p>
<p v-else>Scan completed</p>
<!-- at {{hit.scanned_at}} -->
</div>
<div class='state scan' v-if="hit.scanned_at">
<!-- <h2>analog drawing</h2> -->
<!-- <div class="statebox"> -->
<!-- <img src="fake_scan.jpg">-->
<img :src="hit.scan_image">
<!-- </div> -->
</div>
</template>
<!-- <div class="phase" id="waiting_for_human"> <!-- <div class="phase" id="waiting_for_human">
<span class="narrative_phase_text">waiting for human worker to accept task</span> <span class="narrative_phase_text">waiting for human worker to accept task</span>
@ -121,10 +17,10 @@
<div class="phase" id="human_accepted_task"> <div class="phase" id="human_accepted_task">
<span class="narrative_phase_text">task accepted by human worker</span> <span class="narrative_phase_text">task accepted by human worker</span>
</div> --> </div> -->
<!--
<div class="phase" id="worker_specs"> <div class="phase" id="worker_specs">
<span class="grid-item spec_name" id="hit_id_descriptor">human intelligent task id</span> <span class="grid-item spec_name" id="hit_id_descriptor">human intelligent task id</span>
<span class="grid-item spec_value" id="hit_id">{{hit.id}}</span> <span class="grid-item spec_value" id="hit_id">&nbsp;</span>
<span class="grid-item spec_name" id="worker_id_descriptor">human worker id</span> <span class="grid-item spec_name" id="worker_id_descriptor">human worker id</span>
<span class="grid-item spec_value" id="worker_id">&nbsp;</span> <span class="grid-item spec_value" id="worker_id">&nbsp;</span>
@ -147,13 +43,14 @@
<span class="grid-item spec_name" id="elapsed_time_descriptor">time elapsed</span> <span class="grid-item spec_name" id="elapsed_time_descriptor">time elapsed</span>
<span class="grid-item spec_value" id="elapsed_time">&nbsp;</span> <span class="grid-item spec_value" id="elapsed_time">&nbsp;</span>
</div>-->
</div> </div>
</div> </div>
<script src="script.js"></script> <script src="script.js"></script>

137
www/worker_specs/script.js
View file

@ -1,40 +1,38 @@
// DOM STUFF /////////////////////////////////////////////////////////////////// // DOM STUFF ///////////////////////////////////////////////////////////////////
var app = new Vue({ let divs = {},
el: '#wrapper', spec_names = [
data: { 'worker_id',
message: 'Hello Vue!', 'ip',
hits: { 'location',
'browser',
'os',
'state',
'fee',
'hit_created',
'hit_opened',
'hit_submitted',
'elapsed_time',
'hit_id'
]
divs.linkDOM = function(name){
divs[name] = document.getElementById(`${name}`)
} }
},
methods: { spec_names.forEach(function(name){
formatPrice(value) { divs.linkDOM(name)
let val = (value/1).toFixed(2).replace('.', ',')
return val.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ".")
},
duration(date1, date2){
let s1 = Date.parse(date1) / 1000;
let s2 = Date.parse(date2) / 1000;
let interval = s1 - s2;
let minutes = Math.floor(interval / 60);
let seconds = interval % 60;
let o = `${seconds}″`;
if( minutes > 0) {
o = `${minutes}′` + o;
}
return o;
}
}
// watch: {
// hits: {
// deep: true
// }
// }
}) })
let request_time = timeStamp(),
hit_started = false,
elapsed_time,
hit_finished = false
// SOCKET STUFF //////////////////////////////////////////////////////////////// // SOCKET STUFF ////////////////////////////////////////////////////////////////
@ -45,29 +43,76 @@ ws.addEventListener('open', () => {
// ws.send('hi server') // ws.send('hi server')
}) })
ws.addEventListener('message', (event) => { ws.addEventListener('message', (event) => {
console.log('message: ' + event.data) console.log('message: ' + event.data)
let hits = JSON.parse(event.data) let data = JSON.parse(event.data)
let a = {}; if(data.property === 'hit_opened') {
for(let hitid in app.hits) { if(data.value != null){
a[hitid] = app.hits[hitid]; hit_started = true
hit_finished = false
request_time = new Date()
divs[data.property].innerHTML = `${request_time.format('dd mmm HH:MM:ss')}`
}else{
divs[data.property].innerHTML = `&mdash;`
hit_started = false
} }
for(let hit of hits){
a[hit.id] = hit;
} }
app.hits = a; else if(data.property === 'hit_submitted'){
hit_finished = true;
}
else if(divs[data.property]){
data.value === null ? divs[data.property].innerHTML = `&mdash;` : divs[data.property].innerHTML = `${data.value}`
}
}) })
// ANIMATION STUFF ///////////////////////////////////////////////////////////// // ANIMATION STUFF /////////////////////////////////////////////////////////////
//
//function update(step){ let frames,
// frames_per_sec = 10,
// if(!hit_finished) elapsed_time = `${new Date((Date.now() - request_time)).format('MM"m "ss"s"')}` current_frame = 0
// if(hit_started){
// divs['elapsed_time'].innerHTML = elapsed_time
// }else{
// divs['elapsed_time'].innerHTML = `&mdash;` function makeAnimation(){
// } let now,
//} delta = 0,
last = timeStamp(),
step = 1/frames_per_sec
function frame() {
now = timeStamp()
delta += Math.min(1, (now - last) / 1000)
while(delta > step){
delta -= step
update(step)
}
last = now
requestAnimationFrame(frame)
}
requestAnimationFrame(frame)
}
function update(step){
if(!hit_finished) elapsed_time = `${new Date((Date.now() - request_time)).format('MM"m "ss"s"')}`
if(hit_started){
divs['elapsed_time'].innerHTML = elapsed_time
}else{
divs['elapsed_time'].innerHTML = `&mdash;`
}
}
makeAnimation()
function timeStamp(){return window.performance && window.performance.now ? window.performance.now() : new Date().getTime()}

134
www/worker_specs/style.css
View file

@ -12,20 +12,18 @@
:root{ :root{
--base-font-size: 17px; --base-font-size: 17px;
--big-font-size: calc(var(--base-font-size)*1.5); --spec_name-font-size: 120%;
--bigger-font-size: calc(var(--big-font-size)*1.5); --spec_value-font-size: 250%;
/* --spec_name-font-size: 120%;
--spec_value-font-size: 250%; */
--base-color: #271601; /* tekst */ --base-color: #271601; /* tekst */
--alt-color: #FFF5DF; /* achtergrond */ --alt-color: #FFF5DF; /* achtergrond */
/* /////// GAT VOORKANT PLEK & POSITIE //////// */ /* /////// GAT VOORKANT PLEK & POSITIE //////// */
/* */ /* */ /* */ /* */
/* */ --pos-x: 557px; /* */ /* */ --pos-x: 555px; /* */
/* */ --pos-y: 110px; /* */ /* */ --pos-y: 110px; /* */
/* */ --width: 420px; /* 115mm */ /* */ --width: 420px; /* 115mm */
/* */ --height: 970px; /* 270mm */ /* */ --height: 1000px; /* 270mm */
/* */ /* */ /* */ /* */
/* //////////////////////////////////////////// */ /* //////////////////////////////////////////// */
@ -49,9 +47,9 @@ html, body{
position: absolute; position: absolute;
left: var(--pos-x); left: var(--pos-x);
bottom: calc(100vh - (var(--pos-y) + var(--height))); top: var(--pos-y);
width: var(--width); width: var(--width);
height: auto; height: var(--height);
background: var(--alt-color); background: var(--alt-color);
box-sizing: border-box; box-sizing: border-box;
@ -59,109 +57,6 @@ html, body{
} }
#wrapper .hit{
display:block;
}
.transition{
overflow:hidden;
display: block;
position:relative;
padding-left: calc(50% + 20px);
font-size: var(--base-font-size);
height: 40px;
/* text-align: center; */
animation-duration: 3s;
animation-name: slidein;
margin: 10px 0px;
}
.transition p{
/* height: 100%; */
margin: 12px 2px 2px 2px;
}
.transition.no_arrow{
padding: 0;
text-align: center;
}
.transition:not(.no_arrow)::before{
content:'⇩'; /*'⇓';*/
display:block;
position:absolute;
font-family:'monospace';
font-size: 80px;
top:10px;
left:calc(50% - 24px);
line-height:0;
}
.state{
overflow:hidden;
animation-duration: 3s;
animation-name: slidein;
transition: max-height 1s;
/* max-height: 200px; */
margin: 2px;
border:solid 2px black;
padding: 12px;
}
.state h2{
font-family: 'bebas';
font-size: var(--bigger-font-size);
margin: 2px;
font-weight: normal;
}
.state p{
margin: 0;
font-family: 'bebas';
font-size: var(--big-font-size);
opacity: 1;
transition: opacity .5s;
}
.state img{
max-width: 100%;
}
.state .descriptor{
font-family: 'freesans';
font-size: var(--base-font-size);
margin-top: calc(var(--base-font-size)/2);
}
.state.assignment-hidden {
/*On abandon etc.*/
max-height: 0px;
opacity: 0;
}
@keyframes slidein {
from {
max-height:0;
}
to {
max-height: 600px;
}
}
/*
#worker_specs{ #worker_specs{
display:grid; display:grid;
grid-template-columns: 1fr ; grid-template-columns: 1fr ;
@ -202,20 +97,3 @@ html, body{
position: relative; position: relative;
top: 5px; top: 5px;
} }
*/
.columns{
display: flex;
flex-direction: row;
}
.columns .column{
flex-grow: 1;
width: 50%;
}
.scan img{
vertical-align:bottom;
}