hugvey/hugvey/tools.py

import logging
import yaml
import os
import json
from hugvey.voice import VoiceStorage

logger = logging.getLogger('toolbox')

# From https://stackoverflow.com/a/1165552
class DictDiffer(object):
    """
    Calculate the difference between two dictionaries as:
    (1) items added
    (2) items removed
    (3) keys same in both but changed values
    (4) keys same in both and unchanged values
    """
    def __init__(self, current_dict, past_dict):
        self.current_dict, self.past_dict = current_dict, past_dict
        self.set_current, self.set_past = set(current_dict.keys()), set(past_dict.keys())
        self.intersect = self.set_current.intersection(self.set_past)
    def added(self):
        return self.set_current - self.intersect 
    def removed(self):
        return self.set_past - self.intersect 
    def changed(self):
        return set(o for o in self.intersect if self.past_dict[o] != self.current_dict[o])
    def unchanged(self):
        return set(o for o in self.intersect if self.past_dict[o] == self.current_dict[o])

class Toolbox:
    def __init__(self, configFile):
        self.languageFiles = {}
        self.languageConfig = {}
        
        with open(configFile, 'r') as fp:
            logger.debug('Load config from {}'.format(configFile))
            self.config = yaml.safe_load(fp)

        self.hugvey_ids = [i + 1 for i in range(self.config['hugveys'])]

        self.loadLanguages()
        
        
        voice_dir = os.path.join(self.config['web']['files_dir'], 'voices')
        self.voiceStorage = VoiceStorage(voice_dir, self.languageConfig)
        
    
    def loadLanguages(self):
        logger.debug('load language files')
        self.languages = {}

        for lang in self.config['languages']:
            lang_filename = os.path.join(self.config['web']['files_dir'], lang['file'])
            self.languageFiles[lang['code']] = lang['file']
            self.languageConfig[lang['code']] = lang
            with open(lang_filename, 'r') as fp:
                self.languages[lang['code']] = json.load(fp)
                
            if lang['token'] == 'LB_TOKEN' or lang['token'] == 'SECRET_KEY':
                raise Exception("Are you using the right config file? Language key not configured properly!")

    def get_audio_filenames(self):
        """
        Get all audio files as defined trough the config.
        """
        filenames = [
            'local/crash.wav'
            ]
        for langCode in self.languages:
            logger.info(f'lang {langCode}')
            msgs = [node for node in self.languages[langCode] if node['@type'] == 'Msg']
            for msg in msgs:
                if 'audio' in msg and msg['audio'] is not None:
                    filenames.append(msg['audio']['file'])
                    continue
                
                if '$' in msg['text']:
                    # skip variable texts
                    continue
                
                fn = self.voiceStorage.getFilename(langCode, msg['text'], False)
                filenames.append(fn)
        
        return filenames
    
    def get_existing_filesnames(self):
        existing_files = []
        for path, subdirs, files in os.walk(self.config['web']['files_dir']):
            for name in files:
                if name[-4:] == '.wav':
                    existing_files.append(os.path.join(path, name))
        return existing_files
    
    def clean_audio_files(self):
        needed_files = self.get_audio_filenames()
        existing_files = self.get_existing_filesnames()
        
#         if 'local/voices/en-GB/static/9c/9ce29fe21fa813cca9db94419947238f6f215da1.wav' in needed_files:
#             print("GOOO!")
#         else:
#             print('ojee')
#         exit()
        
        for fn in existing_files:
            if fn not in needed_files:
                logger.warn(f"Remove {fn}")
                os.unlink(fn)
            else:
                logger.debug(f"Keep {fn}")
        
        missingFiles = []
        for fn in needed_files:
            if fn not in existing_files:
                missingFiles.append(fn)
#                 logger.info(f"Missing {fn}")

        logger.info("{} files missing".format(len(missingFiles)))
                
    
    def fix_story_file(self, lang_code):
        if lang_code not in self.languages.keys():
            logger.critical("Invalid langauge code")
            logger.warn(f"Valid codes are {' '.join(self.languages.keys())}")
            return
        
        
        filename = os.path.join(self.config['web']['files_dir'],self.languageFiles[lang_code])
        story = self.languages[lang_code]
        
        beginnings = [item for item in story if 'beginning' in item and item['beginning'] is True]
        if len(beginnings) < 1:
            logger.critical("No beginning set")
        if len(beginnings) > 1:
            logger.critical(f"{len(beginnings)} beginning messages configured. Set only one")
        
        itemsPerId = {item['@id']: item for item in story}
        
        for i, item in enumerate(story):
            if item['@type'] == 'Direction':
                if type(item['source']) == dict:
                    logger.warn(f"Fixing broken direction {item['@id']}, please check if everything still works!")
                    validMsg = itemsPerId[item['source']['@id']]
                    diff = DictDiffer(item['source'], validMsg)
                    if diff.changed() or diff.added() or diff.removed():
                        logger.warn("Changes found between messages")
                        logger.warn(f"Changed: {list(diff.changed())} Keys that will be remove: {list(diff.added())} Keys that will be added: {list(diff.removed())}")
                        logger.info(f"Direction pointed to {item['source']}")
                        logger.info(f"Will now point to {validMsg}")
                    item['source'] = item['source']['@id']
        
        with open(filename, 'w') as fp:
            json.dump(story, fp, indent=2)
            logger.info(f"Wrote to {filename}")