Add tool foor adding files to git

hugvey/tools.py

@@ -0,0 +1,99 @@
+import logging
+import yaml
+import os
+import json
+from hugvey.voice import VoiceStorage
+
+logger = logging.getLogger('toolbox')
+
+class Toolbox:
+    def __init__(self, configFile):
+        self.languageFiles = {}
+        self.languageConfig = {}
+        
+        with open(configFile, 'r') as fp:
+            logger.debug('Load config from {}'.format(configFile))
+            self.config = yaml.safe_load(fp)
+
+        self.hugvey_ids = [i + 1 for i in range(self.config['hugveys'])]
+
+        self.loadLanguages()
+        
+        
+        voice_dir = os.path.join(self.config['web']['files_dir'], 'voices')
+        self.voiceStorage = VoiceStorage(voice_dir, self.languageConfig)
+        
+    
+    def loadLanguages(self):
+        logger.debug('load language files')
+        self.languages = {}
+
+        for lang in self.config['languages']:
+            lang_filename = os.path.join(self.config['web']['files_dir'], lang['file'])
+            self.languageFiles[lang['code']] = lang['file']
+            self.languageConfig[lang['code']] = lang
+            with open(lang_filename, 'r') as fp:
+                self.languages[lang['code']] = json.load(fp)
+                
+            if lang['token'] == 'LB_TOKEN' or lang['token'] == 'SECRET_KEY':
+                raise Exception("Are you using the right config file? Language key not configured properly!")
+
+    def get_audio_filenames(self):
+        """
+        Get all audio files as defined trough the config.
+        """
+        filenames = [
+            'local/crash.wav'
+            ]
+        for langCode in self.languages:
+            logger.info(f'lang {langCode}')
+            msgs = [node for node in self.languages[langCode] if node['@type'] == 'Msg']
+            for msg in msgs:
+                if 'audio' in msg and msg['audio'] is not None:
+                    filenames.append(msg['audio']['file'])
+                    continue
+                
+                if '$' in msg['text']:
+                    # skip variable texts
+                    continue
+                
+                fn = self.voiceStorage.getFilename(langCode, msg['text'], False)
+                filenames.append(fn)
+        
+        return filenames
+    
+    def get_existing_filesnames(self):
+        existing_files = []
+        for path, subdirs, files in os.walk(self.config['web']['files_dir']):
+            for name in files:
+                if name[-4:] == '.wav':
+                    existing_files.append(os.path.join(path, name))
+        return existing_files
+    
+    def clean_audio_files(self):
+        needed_files = self.get_audio_filenames()
+        existing_files = self.get_existing_filesnames()
+        
+#         if 'local/voices/en-GB/static/9c/9ce29fe21fa813cca9db94419947238f6f215da1.wav' in needed_files:
+#             print("GOOO!")
+#         else:
+#             print('ojee')
+#         exit()
+        
+        for fn in existing_files:
+            if fn not in needed_files:
+                logger.warn(f"Remove {fn}")
+                os.unlink(fn)
+            else:
+                logger.debug(f"Keep {fn}")
+        
+        missingFiles = []
+        for fn in needed_files:
+            if fn not in existing_files:
+                missingFiles.append(fn)
+#                 logger.info(f"Missing {fn}")
+
+        logger.info("{} files missing".format(len(missingFiles)))
+                
+    
+        

hugvey_client.py

@@ -48,7 +48,6 @@ if __name__ == '__main__':
         host = urlparse(hv.config['events']['cmd_address']).hostname
         logger.info("Connect to logger on {}".format(host))
         socket_handler = logging.handlers.SocketHandler(host, 19996)  # default listening address
-        socket_handler.setLevel(logging.DEBUG) # don't send BS messages
         logger.addHandler(socket_handler);
         
     hv.start()

tools.py

@@ -0,0 +1,38 @@
+from hugvey.client import Hugvey
+import logging
+import logging.handlers
+import coloredlogs
+import argparse
+from urllib.parse import urlparse
+from hugvey.tools import Toolbox
+
+if __name__ == '__main__':
+    argParser = argparse.ArgumentParser(description='Start up a Hugvey pillow. Mic stream becomes available on TCP Socket, and starts listening + emitting events')
+    argParser.add_argument(
+            '--config',
+            '-c',
+            required=True,
+            type=str,
+            help='The yaml config file to load'
+        )
+    argParser.add_argument(
+            '--files',
+            action='store_true',
+            help="Add/remove new & unused file"
+        )
+
+    args = argParser.parse_args()
+
+    coloredlogs.install(
+        level=logging.DEBUG
+    )
+    
+    logger = logging.getLogger("toolbox")
+#     logger.setLevel(1)  # to send all records to cutelog
+    
+
+    tools = Toolbox(args.config)
+    if args.files:
+        logger.info("Filenames")
+        tools.clean_audio_files()
+