import logging import yaml import os import json from hugvey.voice import VoiceStorage logger = logging.getLogger('toolbox') # From https://stackoverflow.com/a/1165552 class DictDiffer(object): """ Calculate the difference between two dictionaries as: (1) items added (2) items removed (3) keys same in both but changed values (4) keys same in both and unchanged values """ def __init__(self, current_dict, past_dict): self.current_dict, self.past_dict = current_dict, past_dict self.set_current, self.set_past = set(current_dict.keys()), set(past_dict.keys()) self.intersect = self.set_current.intersection(self.set_past) def added(self): return self.set_current - self.intersect def removed(self): return self.set_past - self.intersect def changed(self): return set(o for o in self.intersect if self.past_dict[o] != self.current_dict[o]) def unchanged(self): return set(o for o in self.intersect if self.past_dict[o] == self.current_dict[o]) class Toolbox: def __init__(self, configFile): self.languageFiles = {} self.languageConfig = {} with open(configFile, 'r') as fp: logger.debug('Load config from {}'.format(configFile)) self.config = yaml.safe_load(fp) self.hugvey_ids = [i + 1 for i in range(self.config['hugveys'])] self.loadLanguages() voice_dir = os.path.join(self.config['web']['files_dir'], 'voices') self.voiceStorage = VoiceStorage(voice_dir, self.languageConfig) def loadLanguages(self): logger.debug('load language files') self.languages = {} for lang in self.config['languages']: lang_filename = os.path.join(self.config['web']['files_dir'], lang['file']) self.languageFiles[lang['code']] = lang['file'] self.languageConfig[lang['code']] = lang with open(lang_filename, 'r') as fp: self.languages[lang['code']] = json.load(fp) if lang['token'] == 'LB_TOKEN' or lang['token'] == 'SECRET_KEY': raise Exception("Are you using the right config file? Language key not configured properly!") def get_audio_filenames(self): """ Get all audio files as defined trough the config. """ filenames = [ 'local/crash.wav' ] for langCode in self.languages: logger.info(f'lang {langCode}') msgs = [node for node in self.languages[langCode] if node['@type'] == 'Msg'] for msg in msgs: if 'audio' in msg and msg['audio'] is not None: filenames.append(msg['audio']['file']) continue if '$' in msg['text']: # skip variable texts continue fn = self.voiceStorage.getFilename(langCode, msg['text'], False) filenames.append(fn) return filenames def get_existing_filesnames(self): existing_files = [] for path, subdirs, files in os.walk(self.config['web']['files_dir']): for name in files: if name[-4:] == '.wav': existing_files.append(os.path.join(path, name)) return existing_files def clean_audio_files(self): needed_files = self.get_audio_filenames() existing_files = self.get_existing_filesnames() # if 'local/voices/en-GB/static/9c/9ce29fe21fa813cca9db94419947238f6f215da1.wav' in needed_files: # print("GOOO!") # else: # print('ojee') # exit() for fn in existing_files: if fn not in needed_files: logger.warn(f"Remove {fn}") os.unlink(fn) else: logger.debug(f"Keep {fn}") missingFiles = [] for fn in needed_files: if fn not in existing_files: missingFiles.append(fn) # logger.info(f"Missing {fn}") logger.info("{} files missing".format(len(missingFiles))) def fix_story_file(self, lang_code): if lang_code not in self.languages.keys(): logger.critical("Invalid langauge code") logger.warn(f"Valid codes are {' '.join(self.languages.keys())}") return filename = os.path.join(self.config['web']['files_dir'],self.languageFiles[lang_code]) story = self.languages[lang_code] beginnings = [item for item in story if 'beginning' in item and item['beginning'] is True] if len(beginnings) < 1: logger.critical("No beginning set") if len(beginnings) > 1: logger.critical(f"{len(beginnings)} beginning messages configured. Set only one") itemsPerId = {item['@id']: item for item in story} for i, item in enumerate(story): if item['@type'] == 'Direction': if type(item['source']) == dict: logger.warn(f"Fixing broken direction {item['@id']}, please check if everything still works!") validMsg = itemsPerId[item['source']['@id']] diff = DictDiffer(item['source'], validMsg) if diff.changed() or diff.added() or diff.removed(): logger.warn("Changes found between messages") logger.warn(f"Changed: {list(diff.changed())} Keys that will be remove: {list(diff.added())} Keys that will be added: {list(diff.removed())}") logger.info(f"Direction pointed to {item['source']}") logger.info(f"Will now point to {validMsg}") item['source'] = item['source']['@id'] with open(filename, 'w') as fp: json.dump(story, fp, indent=2) logger.info(f"Wrote to {filename}")