Fetch voice from Lyrebird

hugvey/central_command.py

@@ -20,6 +20,7 @@ import json
 import logging
 import queue
 import threading
+from hugvey.voice import VoiceStorage
 
 
 logger = logging.getLogger("command")
@@ -73,6 +74,7 @@ class CentralCommand(object):
 
         self.loadLanguages()
         
+        
         self.panopticon = Panopticon(self, self.config)
     
     def loadLanguages(self):
@@ -122,6 +124,7 @@ class CentralCommand(object):
         """
         prepare command to be picked up by the sender
         """
+        logging.debug(f"COmmand {hv_id}: {msg}")
         if threading.current_thread().getName() != 'MainThread':
             # Threading nightmares! Adding to queue from other thread/loop (not sure which is the isse)
             # won't trigger asyncios queue.get() so we have to do this thread
@@ -226,9 +229,9 @@ class CentralCommand(object):
 
         self.tasks = {}  # collect tasks so we can cancel in case of error
         self.tasks['eventListener'] = self.loop.create_task(
-            self.eventListener())
+            self.catchException(self.eventListener()))
         self.tasks['commandSender'] = self.loop.create_task(
-            self.commandSender())
+            self.catchException(self.commandSender()))
 
         # we want the web interface in a separate thread
         self.panopticon_thread = threading.Thread(
@@ -240,6 +243,14 @@ class CentralCommand(object):
     def stop(self):
         self.isRunning.clear()
     
+    async def catchException(self, awaitable):
+        try:
+            print(awaitable)
+            await awaitable
+        except Exception as e:
+            logger.exception(e)
+            logger.critical(f"Hugvey restart might be required but not implemented yet")
+
 
 class HugveyState(object):
     """Represents the state of a Hugvey client on the server.
@@ -309,6 +320,7 @@ class HugveyState(object):
 
     async def catchException(self, awaitable):
         try:
+            print(awaitable)
             await awaitable
         except Exception as e:
             logger.exception(e)
@@ -436,7 +448,8 @@ class HugveyState(object):
             await self.isRunning.wait()
             
             # new story instance on each run
-            self.story = Story(self)
+            port = self.command.config['web']['port']
+            self.story = Story(self, port)
             startMsgId = self.startMsgId
             self.startMsgId = None # use only once, reset before 'run'
             self.logger.warn(f"Starting from {startMsgId}")

hugvey/communication.py

@@ -11,6 +11,7 @@ def getTopic(hugvey_id):
 
 
 def zmqSend(socket, hugvey_id, msg):
+    logger.info("SEND: {}".format(msg))
     msgData = json.dumps(msg)
     topic = getTopic(hugvey_id)
     logger.info("Send 0mq to {} containing {}".format(topic, msg))

hugvey/panopticon.py

@@ -16,6 +16,7 @@ import asyncio
 import json
 from urllib.parse import urlparse
 from hugvey import central_command
+from hugvey.voice import VoiceStorage
 
 logger = logging.getLogger("panopticon")
 
@@ -165,15 +166,41 @@ def getUploadHandler(central_command):
             
     return UploadHandler
 
+def getVoiceHandler(voiceStorage):
+    class VoiceHandler(tornado.web.RequestHandler):
+        async def get(self):
+            # TODO: we should be using ZMQ here...
+            text = self.get_argument('text')
+            isVariable = True if int(self.get_argument('variable')) >0 else False
+            fn = await voiceStorage.requestFile(text, isVariable)
+            if not fn:
+                raise Exception(f"No Filename for text: {text}")
+            
+            if int(self.get_argument('filename')) == 1:
+                self.set_header("Content-Type","text/plain")
+                self.write(fn)
+            else:
+                self.set_header("Content-Type","audio/wave")
+                with open(fn, 'rb') as fp:
+                    self.write(fp.read())
+                self.finish()
+    return VoiceHandler 
+            
+
 class Panopticon(object):
     def __init__(self, central_command, config):
         self.command = central_command
         self.config = config
+        
+        voice_dir = os.path.join(self.config['web']['files_dir'], 'voices')
+        self.voiceStorage = VoiceStorage(voice_dir, self.config['voice']['token'])
+        
         self.application = tornado.web.Application([
             (r"/ws", getWebSocketHandler(self.command)),
             (r"/local/(.*)", NonCachingStaticFileHandler,
                 {"path": config['web']['files_dir']}),
             (r"/upload", getUploadHandler(self.command)),
+            (r"/voice", getVoiceHandler(self.voiceStorage)),
             (r"/(.*)", tornado.web.StaticFileHandler,
                 {"path": web_dir, "default_filename": 'index.html'}),
         ], debug=True)

hugvey/story.py

@@ -3,7 +3,9 @@ import time
 import logging
 import re
 import asyncio
+import urllib.parse
 from .communication import LOG_BS
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
 
 logger = logging.getLogger("narrative")
@@ -54,6 +56,9 @@ class Message(object):
         """
         self.variables = re.findall('\$(\w+)', self.text)
     
+    def hasVariables(self) -> bool:
+        return len(self.variables) > 0
+    
     def setReply(self, reply):
         self.reply = reply
 
@@ -83,6 +88,25 @@ class Message(object):
             'replyText': None if self.reply is None else [u.text for u in self.reply.utterances]
         }
         
+    async def getAudioFilePath(self,story):
+        client = AsyncHTTPClient()
+        queryString = urllib.parse.urlencode({
+            'text': self.text,
+            'filename': 1,
+            'variable': 1 if self.hasVariables() else 0
+            })
+        request = HTTPRequest(
+            url = f"http://localhost:{story.panopticon_port}/voice?{queryString}",
+            method="GET"
+            )
+        logger.log(LOG_BS, request.url)
+        response = await client.fetch(request)
+        if response.code != 200:
+            logger.critical(f"Error when fetching filename: {response.code} for {queryString}")
+            return None
+        
+        return response.body.decode().strip()
+
 
 class Reply(object):
     def __init__(self, message: Message):
@@ -363,9 +387,10 @@ class Story(object):
     # TODO should we separate 'narrative' (the graph) from the story (the
     # current user flow)
 
-    def __init__(self, hugvey_state):
+    def __init__(self, hugvey_state, panopticon_port):
         super(Story, self).__init__()
         self.hugvey = hugvey_state
+        self.panopticon_port = panopticon_port
 
         self.events = []  # queue of received events
         self.commands = []  # queue of commands to send
@@ -485,7 +510,7 @@ class Story(object):
         if self.isRunning:
             self.isRunning = False
 
-    def _processPendingEvents(self):
+    async def _processPendingEvents(self):
         # Gather events:
         nr = len(self.events)
         for i in range(nr):
@@ -497,7 +522,7 @@ class Story(object):
                 # a client connected. Should only happen in the beginning or in case of error
                 # that is, until we have a 'reset' or 'start' event.
                 # reinitiate current message
-                self.setCurrentMessage(self.currentMessage)
+                await self.setCurrentMessage(self.currentMessage)
 
             if e['event'] == "playbackFinish":
                 if e['msgId'] == self.currentMessage.id:
@@ -518,7 +543,7 @@ class Story(object):
                         logger.warn("Interrupt message, replay {}".format(self.previousReply.forMessage.id))
                         self.currentReply = self.previousReply
                         self.previousReply.forMessage.interruptCount += 1
-                        self.currentMessage = self.setCurrentMessage(self.previousReply.forMessage)
+                        self.currentMessage = await self.setCurrentMessage(self.previousReply.forMessage)
                     
                 # log if somebody starts speaking
                 # TODO: implement interrupt
@@ -532,7 +557,7 @@ class Story(object):
                     utterance.setFinished(self.timer.getElapsed())
                     
         
-    def _processDirections(self, directions):
+    async def _processDirections(self, directions):
         for direction in directions:
             for condition in direction.conditions:
                 if condition.isMet(self):
@@ -542,7 +567,7 @@ class Story(object):
                     self.addToLog(condition)
                     self.addToLog(direction)
                     self.currentMessage.setFinished(self.timer.getElapsed())
-                    self.setCurrentMessage(direction.msgTo)
+                    await self.setCurrentMessage(direction.msgTo)
                     return direction
     
     def addToLog(self, node):
@@ -563,13 +588,13 @@ class Story(object):
             await self.timer.isRunning.wait() # wait for un-pause
 
             for i in range(len(self.events)):
-                self._processPendingEvents()
+                await self._processPendingEvents()
 
                 if self.currentMessage.id not in self.directionsPerMsg:
                     self.finish()
 
             directions = self.getCurrentDirections()
-            self._processDirections(directions)
+            await self._processDirections(directions)
 
             # TODO create timer event
             # self.commands.append({'msg':'TEST!'})
@@ -581,7 +606,7 @@ class Story(object):
 
         logger.info("Stop renderer")
 
-    def setCurrentMessage(self, message):
+    async def setCurrentMessage(self, message):
         if self.currentMessage and not self.lastMsgFinishTime:
             logger.info("Interrupt playback {}".format(self.currentMessage.id))
             # message is playing
@@ -606,18 +631,12 @@ class Story(object):
             message.id, message.text))
         self.addToLog(message)
         # TODO: prep events & timer etc.
-        if message.audioFile:
+        # TODO: preload file paths if no variables are set, or once these are loaded
-            self.hugvey.sendCommand({
+        self.hugvey.sendCommand({
-                'action': 'play',
+            'action': 'play',
-                'file': message.audioFile,
+            'file': await message.getAudioFilePath(self),
-                'id': message.id,
+            'id': message.id,
-            })
+        })
-        else:
-            self.hugvey.sendCommand({
-                'action': 'play',
-                'msg': message.text,
-                'id': message.id,
-            })
 
         logger.debug("Pending directions: ")
 
@@ -640,7 +659,7 @@ class Story(object):
             startMsg = self.get(customStartMsgId)
         else:
             startMsg = self.startMessage
-        self.setCurrentMessage(startMsg)
+        await self.setCurrentMessage(startMsg)
         await self._renderer()
     
     def isFinished(self):

hugvey/voice.py

@@ -0,0 +1,80 @@
+import os
+import time
+import json
+import logging
+import threading
+from requests_threads import AsyncSession
+from hashlib import sha1
+import asyncio
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
+
+logger = logging.getLogger("voice")
+
+class VoiceStorage(object):
+    """
+    Store & keep voices that are not part of the story json
+    """
+    def __init__(self, cache_dir, token):
+        self.cache_dir = cache_dir
+        if not os.path.exists(self.cache_dir):
+            raise Exception(f"Cache dir does not exists: {self.cache_dir}")
+#         self.request_session = AsyncSession(n=5)
+        self.pendingRequests = {}
+        self.token = token
+    
+    def getId(self, text):
+        return sha1(text.encode()).hexdigest()
+    
+    def getFilename(self, text, isVariable=False):
+        subdir = 'static' if not isVariable else 'variable'
+        id = self.getId(text)
+        prefix = id[:2]
+        storageDir = os.path.join(self.cache_dir, subdir, prefix)
+        fn = os.path.join(storageDir, f"{id}.wav")
+        return fn
+    
+    async def requestFile(self, text, isVariable=False) -> str:
+        id = self.getId(text)
+        fn = self.getFilename(text)
+        
+        if os.path.exists(fn):
+            return fn
+        
+        if id in self.pendingRequests:
+            await self.pendingRequests[id].wait()
+            if os.path.exists(fn):
+                return fn
+            return None
+        
+        dirname = os.path.dirname(fn)
+        if not os.path.exists(dirname):
+            logger.debug(f"create directory for file: {dirname}")
+            os.makedirs(dirname, exist_ok=True)
+            
+        self.pendingRequests[id] = asyncio.Event()
+        
+        http_client = AsyncHTTPClient()
+        request = HTTPRequest(
+            method="POST",
+            url="https://avatar.lyrebird.ai/api/v0/generate",
+            body=json.dumps({"text": text}),
+            headers={"authorization": f"Bearer {self.token}"}
+        )
+        try:
+            response = await http_client.fetch(request)
+        except Exception as e:
+            logger.exception(e)
+            self.pendingRequests[id].set()
+            return None
+        else:
+            if response.code != 200:
+                logger.critical(f"No proper response! {response.code}")
+                self.pendingRequests[id].set()
+                return None
+            
+            logger.debug(f"Wrote body: {response.code}")
+            with open(fn, "wb") as f:
+                f.write(response.body)
+            self.pendingRequests[id].set()
+            print(type(fn), fn)
+            return fn

server_config.yml

@@ -7,6 +7,7 @@ voice:
     port: 4444
     chunk: 2972
     google_credentials: "../test_googlespeech/My First Project-0c7833e0d5fa.json"
+    token: "oauth_SOMETHING"
 hugveys: 25
 languages:
     - code: en-GB

www/js/hugvey_console.js

@@ -265,6 +265,11 @@ class Graph {
         this.showMsg( this.selectedMsg );
     }
     
+    getAudioUrlForMsg(msg) {
+    	let isVariable = msg['text'].includes('$') ? '1' : '0';
+        return `http://localhost:8888/voice?text=${encodeURIComponent(msg['text'])}&variable=${isVariable}&filename=0`;
+    }
+
     showMsg( msg ) {
         let msgEl = document.getElementById( 'msg' );
         msgEl.innerHTML = "";
@@ -286,20 +291,19 @@ class Graph {
         if ( msg['start'] == true ) {
             startAttributes['checked'] = 'checked';
         }
+        
+        let audioSrcEl = crel('source', {'src': this.getAudioUrlForMsg(msg)});
         let audioSpan = crel(
                 'span',
                 {
                     'title':  msg['audio'] ?  msg['audio']['file'] : "",
                     'class': "label-value",
                 },
-                msg['audio'] ? msg['audio']['original_name'] : "" 
+                crel(
+            		'audio', {'controls': 'controls'},
+            		audioSrcEl
+                ) 
             );
-        if(msg['audio']) {
-            audioSpan.appendChild(crel(
-                    'audio', {'controls': 'controls'},
-                    crel('source', {'src':msg['audio']['file']})
-                    ));
-        }
         let msgInfoEl = crel( 'div', { 'class': 'msg__info' },
             crel('div', {
                 'class':'btn btn--delete',
@@ -319,7 +323,10 @@ class Graph {
                     'name': msg['@id'] + '-text',
                     'value': msg['text'],
                     'on': {
-                        'change': this.getEditEventListener()
+                        'change': this.getEditEventListener(function(){
+                        	audioSrcEl.src = panopticon.graph.getAudioUrlForMsg(msg);
+                        	audioSrcEl.parentElement.load();
+                        })
                     }
                 } )
             ),
@@ -855,7 +862,7 @@ class Graph {
      * Use wrapper method, because for event handlers 'this' will refer to
      * the input object
      */
-    getEditEventListener() {
+    getEditEventListener(callback) {
         let graph = this;
         let el = function( e ) {
             console.info("Changed", e);
@@ -880,6 +887,10 @@ class Graph {
             // node[field] = e.srcElement.value;
 
             graph.build();
+            
+            if(typeof callback !== 'undefined'){
+            	callback();
+            }
         }
         return el;
     }