From 967590981c323591ca427fef90f928ca7482a001 Mon Sep 17 00:00:00 2001
From: Ruben van de Ven <git@rubenvandeven.com>
Date: Tue, 10 Aug 2021 11:49:31 +0200
Subject: [PATCH] pretty printed list-hits

---
 .gitignore              |  2 ++
 exhaust.py              |  8 ++++++++
 exhausting_mturk/api.py | 15 +++++++++++++++
 3 files changed, 25 insertions(+)

diff --git a/.gitignore b/.gitignore
index 92ea88a..e38755a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 config.local.yml
 *.sla
+*.pyc
+
diff --git a/exhaust.py b/exhaust.py
index ef0e579..71fa592 100644
--- a/exhaust.py
+++ b/exhaust.py
@@ -40,6 +40,12 @@ if __name__ == "__main__":
         action='store_true',
         help='fetch HIT status and assignments from MTurk'
     )
+    argParser.add_argument(
+        '--list-hits',
+        '-lh',
+        action='store_true',
+        help='For debugging: list all hits using API'
+    )
     argParser.add_argument(
         '--for-real',
         action='store_true',
@@ -67,6 +73,8 @@ if __name__ == "__main__":
 
     if args.update:
         connection.load_new_submissions()
+    if args.list_hits:
+        connection.list_hits()
     elif args.publish:
         if args.publish not in batch_files:
             raise Exception(f'Not a valid batch. Use one of {batches=}')
diff --git a/exhausting_mturk/api.py b/exhausting_mturk/api.py
index eafdd74..4b9e88a 100644
--- a/exhausting_mturk/api.py
+++ b/exhausting_mturk/api.py
@@ -38,6 +38,21 @@ class Connection():
            endpoint_url = MTURK_REAL if self.config['for_real'] else MTURK_SANDBOX
         )
 
+    def list_hits(self):
+        next_token = None
+        all = 0
+        while True:
+            logger.info('fetch hits:')
+            r = self.mturk.list_hits() if not next_token else self.mturk.list_hits(NextToken = next_token)
+            print(yaml.dump(r['HITs'], sort_keys=False, default_flow_style=False))
+            all += len(r['HITs'])
+            if 'NextToken' in r and r['NextToken']:
+                next_token = r['NextToken']
+            else:
+                break
+        logger.info(f'Fetched {all} hits')
+
+
     def load_new_submissions(self):
         hit_files = glob.glob('batches/hits/*.json')
         for hit_file in hit_files: