156 lines
6 KiB
Python
156 lines
6 KiB
Python
import logging
|
|
import yaml
|
|
import os
|
|
import json
|
|
from hugvey.voice import VoiceStorage
|
|
|
|
logger = logging.getLogger('toolbox')
|
|
|
|
# From https://stackoverflow.com/a/1165552
|
|
class DictDiffer(object):
|
|
"""
|
|
Calculate the difference between two dictionaries as:
|
|
(1) items added
|
|
(2) items removed
|
|
(3) keys same in both but changed values
|
|
(4) keys same in both and unchanged values
|
|
"""
|
|
def __init__(self, current_dict, past_dict):
|
|
self.current_dict, self.past_dict = current_dict, past_dict
|
|
self.set_current, self.set_past = set(current_dict.keys()), set(past_dict.keys())
|
|
self.intersect = self.set_current.intersection(self.set_past)
|
|
def added(self):
|
|
return self.set_current - self.intersect
|
|
def removed(self):
|
|
return self.set_past - self.intersect
|
|
def changed(self):
|
|
return set(o for o in self.intersect if self.past_dict[o] != self.current_dict[o])
|
|
def unchanged(self):
|
|
return set(o for o in self.intersect if self.past_dict[o] == self.current_dict[o])
|
|
|
|
class Toolbox:
|
|
def __init__(self, configFile):
|
|
self.languageFiles = {}
|
|
self.languageConfig = {}
|
|
|
|
with open(configFile, 'r') as fp:
|
|
logger.debug('Load config from {}'.format(configFile))
|
|
self.config = yaml.safe_load(fp)
|
|
|
|
self.hugvey_ids = [i + 1 for i in range(self.config['hugveys'])]
|
|
|
|
self.loadLanguages()
|
|
|
|
|
|
voice_dir = os.path.join(self.config['web']['files_dir'], 'voices')
|
|
self.voiceStorage = VoiceStorage(voice_dir, self.languageConfig)
|
|
|
|
|
|
def loadLanguages(self):
|
|
logger.debug('load language files')
|
|
self.languages = {}
|
|
|
|
for lang in self.config['languages']:
|
|
lang_filename = os.path.join(self.config['web']['files_dir'], lang['file'])
|
|
self.languageFiles[lang['code']] = lang['file']
|
|
self.languageConfig[lang['code']] = lang
|
|
with open(lang_filename, 'r') as fp:
|
|
self.languages[lang['code']] = json.load(fp)
|
|
|
|
if lang['token'] == 'LB_TOKEN' or lang['token'] == 'SECRET_KEY':
|
|
raise Exception("Are you using the right config file? Language key not configured properly!")
|
|
|
|
def get_audio_filenames(self):
|
|
"""
|
|
Get all audio files as defined trough the config.
|
|
"""
|
|
filenames = [
|
|
'local/crash.wav'
|
|
]
|
|
for langCode in self.languages:
|
|
logger.info(f'lang {langCode}')
|
|
msgs = [node for node in self.languages[langCode] if node['@type'] == 'Msg']
|
|
for msg in msgs:
|
|
if 'audio' in msg and msg['audio'] is not None:
|
|
filenames.append(msg['audio']['file'])
|
|
continue
|
|
|
|
if '$' in msg['text']:
|
|
# skip variable texts
|
|
continue
|
|
|
|
fn = self.voiceStorage.getFilename(langCode, msg['text'], False)
|
|
filenames.append(fn)
|
|
|
|
return filenames
|
|
|
|
def get_existing_filesnames(self):
|
|
existing_files = []
|
|
for path, subdirs, files in os.walk(self.config['web']['files_dir']):
|
|
for name in files:
|
|
if name[-4:] == '.wav':
|
|
existing_files.append(os.path.join(path, name))
|
|
return existing_files
|
|
|
|
def clean_audio_files(self):
|
|
needed_files = self.get_audio_filenames()
|
|
existing_files = self.get_existing_filesnames()
|
|
|
|
# if 'local/voices/en-GB/static/9c/9ce29fe21fa813cca9db94419947238f6f215da1.wav' in needed_files:
|
|
# print("GOOO!")
|
|
# else:
|
|
# print('ojee')
|
|
# exit()
|
|
|
|
for fn in existing_files:
|
|
if fn not in needed_files:
|
|
logger.warn(f"Remove {fn}")
|
|
os.unlink(fn)
|
|
else:
|
|
logger.debug(f"Keep {fn}")
|
|
|
|
missingFiles = []
|
|
for fn in needed_files:
|
|
if fn not in existing_files:
|
|
missingFiles.append(fn)
|
|
# logger.info(f"Missing {fn}")
|
|
|
|
logger.info("{} files missing".format(len(missingFiles)))
|
|
|
|
|
|
def fix_story_file(self, lang_code):
|
|
if lang_code not in self.languages.keys():
|
|
logger.critical("Invalid langauge code")
|
|
logger.warn(f"Valid codes are {' '.join(self.languages.keys())}")
|
|
return
|
|
|
|
|
|
filename = os.path.join(self.config['web']['files_dir'],self.languageFiles[lang_code])
|
|
story = self.languages[lang_code]
|
|
|
|
beginnings = [item for item in story if 'beginning' in item and item['beginning'] is True]
|
|
if len(beginnings) < 1:
|
|
logger.critical("No beginning set")
|
|
if len(beginnings) > 1:
|
|
logger.critical(f"{len(beginnings)} beginning messages configured. Set only one")
|
|
|
|
itemsPerId = {item['@id']: item for item in story}
|
|
|
|
for i, item in enumerate(story):
|
|
if item['@type'] == 'Direction':
|
|
if type(item['source']) == dict:
|
|
logger.warn(f"Fixing broken direction {item['@id']}, please check if everything still works!")
|
|
validMsg = itemsPerId[item['source']['@id']]
|
|
diff = DictDiffer(item['source'], validMsg)
|
|
if diff.changed() or diff.added() or diff.removed():
|
|
logger.warn("Changes found between messages")
|
|
logger.warn(f"Changed: {list(diff.changed())} Keys that will be remove: {list(diff.added())} Keys that will be added: {list(diff.removed())}")
|
|
logger.info(f"Direction pointed to {item['source']}")
|
|
logger.info(f"Will now point to {validMsg}")
|
|
item['source'] = item['source']['@id']
|
|
|
|
with open(filename, 'w') as fp:
|
|
json.dump(story, fp, indent=2)
|
|
logger.info(f"Wrote to {filename}")
|
|
|
|
|