Fetch voice from Lyrebird
This commit is contained in:
Ruben van de Ven
parent
46941e45a5
commit
79424ccfbf
7 changed files with 186 additions and 34 deletions
|
|
@ -20,6 +20,7 @@ import json
|
|||
import logging
|
||||
import queue
|
||||
import threading
|
||||
from hugvey.voice import VoiceStorage
|
||||
|
||||
|
||||
logger = logging.getLogger("command")
|
||||
|
|
@ -73,6 +74,7 @@ class CentralCommand(object):
|
|||
|
||||
self.loadLanguages()
|
||||
|
||||
|
||||
self.panopticon = Panopticon(self, self.config)
|
||||
|
||||
def loadLanguages(self):
|
||||
|
|
@ -122,6 +124,7 @@ class CentralCommand(object):
|
|||
"""
|
||||
prepare command to be picked up by the sender
|
||||
"""
|
||||
logging.debug(f"COmmand {hv_id}: {msg}")
|
||||
if threading.current_thread().getName() != 'MainThread':
|
||||
# Threading nightmares! Adding to queue from other thread/loop (not sure which is the isse)
|
||||
# won't trigger asyncios queue.get() so we have to do this thread
|
||||
|
|
@ -226,9 +229,9 @@ class CentralCommand(object):
|
|||
|
||||
self.tasks = {} # collect tasks so we can cancel in case of error
|
||||
self.tasks['eventListener'] = self.loop.create_task(
|
||||
self.eventListener())
|
||||
self.catchException(self.eventListener()))
|
||||
self.tasks['commandSender'] = self.loop.create_task(
|
||||
self.commandSender())
|
||||
self.catchException(self.commandSender()))
|
||||
|
||||
# we want the web interface in a separate thread
|
||||
self.panopticon_thread = threading.Thread(
|
||||
|
|
@ -239,6 +242,14 @@ class CentralCommand(object):
|
|||
|
||||
def stop(self):
|
||||
self.isRunning.clear()
|
||||
|
||||
async def catchException(self, awaitable):
|
||||
try:
|
||||
print(awaitable)
|
||||
await awaitable
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
logger.critical(f"Hugvey restart might be required but not implemented yet")
|
||||
|
||||
|
||||
class HugveyState(object):
|
||||
|
|
@ -309,6 +320,7 @@ class HugveyState(object):
|
|||
|
||||
async def catchException(self, awaitable):
|
||||
try:
|
||||
print(awaitable)
|
||||
await awaitable
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
|
|
@ -436,7 +448,8 @@ class HugveyState(object):
|
|||
await self.isRunning.wait()
|
||||
|
||||
# new story instance on each run
|
||||
self.story = Story(self)
|
||||
port = self.command.config['web']['port']
|
||||
self.story = Story(self, port)
|
||||
startMsgId = self.startMsgId
|
||||
self.startMsgId = None # use only once, reset before 'run'
|
||||
self.logger.warn(f"Starting from {startMsgId}")
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ def getTopic(hugvey_id):
|
|||
|
||||
|
||||
def zmqSend(socket, hugvey_id, msg):
|
||||
logger.info("SEND: {}".format(msg))
|
||||
msgData = json.dumps(msg)
|
||||
topic = getTopic(hugvey_id)
|
||||
logger.info("Send 0mq to {} containing {}".format(topic, msg))
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import asyncio
|
|||
import json
|
||||
from urllib.parse import urlparse
|
||||
from hugvey import central_command
|
||||
from hugvey.voice import VoiceStorage
|
||||
|
||||
logger = logging.getLogger("panopticon")
|
||||
|
||||
|
|
@ -165,15 +166,41 @@ def getUploadHandler(central_command):
|
|||
|
||||
return UploadHandler
|
||||
|
||||
def getVoiceHandler(voiceStorage):
|
||||
class VoiceHandler(tornado.web.RequestHandler):
|
||||
async def get(self):
|
||||
# TODO: we should be using ZMQ here...
|
||||
text = self.get_argument('text')
|
||||
isVariable = True if int(self.get_argument('variable')) >0 else False
|
||||
fn = await voiceStorage.requestFile(text, isVariable)
|
||||
if not fn:
|
||||
raise Exception(f"No Filename for text: {text}")
|
||||
|
||||
if int(self.get_argument('filename')) == 1:
|
||||
self.set_header("Content-Type","text/plain")
|
||||
self.write(fn)
|
||||
else:
|
||||
self.set_header("Content-Type","audio/wave")
|
||||
with open(fn, 'rb') as fp:
|
||||
self.write(fp.read())
|
||||
self.finish()
|
||||
return VoiceHandler
|
||||
|
||||
|
||||
class Panopticon(object):
|
||||
def __init__(self, central_command, config):
|
||||
self.command = central_command
|
||||
self.config = config
|
||||
|
||||
voice_dir = os.path.join(self.config['web']['files_dir'], 'voices')
|
||||
self.voiceStorage = VoiceStorage(voice_dir, self.config['voice']['token'])
|
||||
|
||||
self.application = tornado.web.Application([
|
||||
(r"/ws", getWebSocketHandler(self.command)),
|
||||
(r"/local/(.*)", NonCachingStaticFileHandler,
|
||||
{"path": config['web']['files_dir']}),
|
||||
(r"/upload", getUploadHandler(self.command)),
|
||||
(r"/voice", getVoiceHandler(self.voiceStorage)),
|
||||
(r"/(.*)", tornado.web.StaticFileHandler,
|
||||
{"path": web_dir, "default_filename": 'index.html'}),
|
||||
], debug=True)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import time
|
|||
import logging
|
||||
import re
|
||||
import asyncio
|
||||
import urllib.parse
|
||||
from .communication import LOG_BS
|
||||
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
|
||||
|
||||
|
||||
logger = logging.getLogger("narrative")
|
||||
|
|
@ -54,6 +56,9 @@ class Message(object):
|
|||
"""
|
||||
self.variables = re.findall('\$(\w+)', self.text)
|
||||
|
||||
def hasVariables(self) -> bool:
|
||||
return len(self.variables) > 0
|
||||
|
||||
def setReply(self, reply):
|
||||
self.reply = reply
|
||||
|
||||
|
|
@ -82,6 +87,25 @@ class Message(object):
|
|||
'time': None if self.reply is None else [u.startTime for u in self.reply.utterances],
|
||||
'replyText': None if self.reply is None else [u.text for u in self.reply.utterances]
|
||||
}
|
||||
|
||||
async def getAudioFilePath(self,story):
|
||||
client = AsyncHTTPClient()
|
||||
queryString = urllib.parse.urlencode({
|
||||
'text': self.text,
|
||||
'filename': 1,
|
||||
'variable': 1 if self.hasVariables() else 0
|
||||
})
|
||||
request = HTTPRequest(
|
||||
url = f"http://localhost:{story.panopticon_port}/voice?{queryString}",
|
||||
method="GET"
|
||||
)
|
||||
logger.log(LOG_BS, request.url)
|
||||
response = await client.fetch(request)
|
||||
if response.code != 200:
|
||||
logger.critical(f"Error when fetching filename: {response.code} for {queryString}")
|
||||
return None
|
||||
|
||||
return response.body.decode().strip()
|
||||
|
||||
|
||||
class Reply(object):
|
||||
|
|
@ -363,9 +387,10 @@ class Story(object):
|
|||
# TODO should we separate 'narrative' (the graph) from the story (the
|
||||
# current user flow)
|
||||
|
||||
def __init__(self, hugvey_state):
|
||||
def __init__(self, hugvey_state, panopticon_port):
|
||||
super(Story, self).__init__()
|
||||
self.hugvey = hugvey_state
|
||||
self.panopticon_port = panopticon_port
|
||||
|
||||
self.events = [] # queue of received events
|
||||
self.commands = [] # queue of commands to send
|
||||
|
|
@ -485,7 +510,7 @@ class Story(object):
|
|||
if self.isRunning:
|
||||
self.isRunning = False
|
||||
|
||||
def _processPendingEvents(self):
|
||||
async def _processPendingEvents(self):
|
||||
# Gather events:
|
||||
nr = len(self.events)
|
||||
for i in range(nr):
|
||||
|
|
@ -497,7 +522,7 @@ class Story(object):
|
|||
# a client connected. Should only happen in the beginning or in case of error
|
||||
# that is, until we have a 'reset' or 'start' event.
|
||||
# reinitiate current message
|
||||
self.setCurrentMessage(self.currentMessage)
|
||||
await self.setCurrentMessage(self.currentMessage)
|
||||
|
||||
if e['event'] == "playbackFinish":
|
||||
if e['msgId'] == self.currentMessage.id:
|
||||
|
|
@ -518,7 +543,7 @@ class Story(object):
|
|||
logger.warn("Interrupt message, replay {}".format(self.previousReply.forMessage.id))
|
||||
self.currentReply = self.previousReply
|
||||
self.previousReply.forMessage.interruptCount += 1
|
||||
self.currentMessage = self.setCurrentMessage(self.previousReply.forMessage)
|
||||
self.currentMessage = await self.setCurrentMessage(self.previousReply.forMessage)
|
||||
|
||||
# log if somebody starts speaking
|
||||
# TODO: implement interrupt
|
||||
|
|
@ -532,7 +557,7 @@ class Story(object):
|
|||
utterance.setFinished(self.timer.getElapsed())
|
||||
|
||||
|
||||
def _processDirections(self, directions):
|
||||
async def _processDirections(self, directions):
|
||||
for direction in directions:
|
||||
for condition in direction.conditions:
|
||||
if condition.isMet(self):
|
||||
|
|
@ -542,7 +567,7 @@ class Story(object):
|
|||
self.addToLog(condition)
|
||||
self.addToLog(direction)
|
||||
self.currentMessage.setFinished(self.timer.getElapsed())
|
||||
self.setCurrentMessage(direction.msgTo)
|
||||
await self.setCurrentMessage(direction.msgTo)
|
||||
return direction
|
||||
|
||||
def addToLog(self, node):
|
||||
|
|
@ -563,13 +588,13 @@ class Story(object):
|
|||
await self.timer.isRunning.wait() # wait for un-pause
|
||||
|
||||
for i in range(len(self.events)):
|
||||
self._processPendingEvents()
|
||||
await self._processPendingEvents()
|
||||
|
||||
if self.currentMessage.id not in self.directionsPerMsg:
|
||||
self.finish()
|
||||
|
||||
directions = self.getCurrentDirections()
|
||||
self._processDirections(directions)
|
||||
await self._processDirections(directions)
|
||||
|
||||
# TODO create timer event
|
||||
# self.commands.append({'msg':'TEST!'})
|
||||
|
|
@ -581,7 +606,7 @@ class Story(object):
|
|||
|
||||
logger.info("Stop renderer")
|
||||
|
||||
def setCurrentMessage(self, message):
|
||||
async def setCurrentMessage(self, message):
|
||||
if self.currentMessage and not self.lastMsgFinishTime:
|
||||
logger.info("Interrupt playback {}".format(self.currentMessage.id))
|
||||
# message is playing
|
||||
|
|
@ -606,18 +631,12 @@ class Story(object):
|
|||
message.id, message.text))
|
||||
self.addToLog(message)
|
||||
# TODO: prep events & timer etc.
|
||||
if message.audioFile:
|
||||
self.hugvey.sendCommand({
|
||||
'action': 'play',
|
||||
'file': message.audioFile,
|
||||
'id': message.id,
|
||||
})
|
||||
else:
|
||||
self.hugvey.sendCommand({
|
||||
'action': 'play',
|
||||
'msg': message.text,
|
||||
'id': message.id,
|
||||
})
|
||||
# TODO: preload file paths if no variables are set, or once these are loaded
|
||||
self.hugvey.sendCommand({
|
||||
'action': 'play',
|
||||
'file': await message.getAudioFilePath(self),
|
||||
'id': message.id,
|
||||
})
|
||||
|
||||
logger.debug("Pending directions: ")
|
||||
|
||||
|
|
@ -640,7 +659,7 @@ class Story(object):
|
|||
startMsg = self.get(customStartMsgId)
|
||||
else:
|
||||
startMsg = self.startMessage
|
||||
self.setCurrentMessage(startMsg)
|
||||
await self.setCurrentMessage(startMsg)
|
||||
await self._renderer()
|
||||
|
||||
def isFinished(self):
|
||||
|
|
|
|||
80
hugvey/voice.py
Normal file
80
hugvey/voice.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
import os
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
from requests_threads import AsyncSession
|
||||
from hashlib import sha1
|
||||
import asyncio
|
||||
from tornado.httpclient import AsyncHTTPClient, HTTPRequest
|
||||
|
||||
logger = logging.getLogger("voice")
|
||||
|
||||
class VoiceStorage(object):
|
||||
"""
|
||||
Store & keep voices that are not part of the story json
|
||||
"""
|
||||
def __init__(self, cache_dir, token):
|
||||
self.cache_dir = cache_dir
|
||||
if not os.path.exists(self.cache_dir):
|
||||
raise Exception(f"Cache dir does not exists: {self.cache_dir}")
|
||||
# self.request_session = AsyncSession(n=5)
|
||||
self.pendingRequests = {}
|
||||
self.token = token
|
||||
|
||||
def getId(self, text):
|
||||
return sha1(text.encode()).hexdigest()
|
||||
|
||||
def getFilename(self, text, isVariable=False):
|
||||
subdir = 'static' if not isVariable else 'variable'
|
||||
id = self.getId(text)
|
||||
prefix = id[:2]
|
||||
storageDir = os.path.join(self.cache_dir, subdir, prefix)
|
||||
fn = os.path.join(storageDir, f"{id}.wav")
|
||||
return fn
|
||||
|
||||
async def requestFile(self, text, isVariable=False) -> str:
|
||||
id = self.getId(text)
|
||||
fn = self.getFilename(text)
|
||||
|
||||
if os.path.exists(fn):
|
||||
return fn
|
||||
|
||||
if id in self.pendingRequests:
|
||||
await self.pendingRequests[id].wait()
|
||||
if os.path.exists(fn):
|
||||
return fn
|
||||
return None
|
||||
|
||||
dirname = os.path.dirname(fn)
|
||||
if not os.path.exists(dirname):
|
||||
logger.debug(f"create directory for file: {dirname}")
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
|
||||
self.pendingRequests[id] = asyncio.Event()
|
||||
|
||||
http_client = AsyncHTTPClient()
|
||||
request = HTTPRequest(
|
||||
method="POST",
|
||||
url="https://avatar.lyrebird.ai/api/v0/generate",
|
||||
body=json.dumps({"text": text}),
|
||||
headers={"authorization": f"Bearer {self.token}"}
|
||||
)
|
||||
try:
|
||||
response = await http_client.fetch(request)
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
self.pendingRequests[id].set()
|
||||
return None
|
||||
else:
|
||||
if response.code != 200:
|
||||
logger.critical(f"No proper response! {response.code}")
|
||||
self.pendingRequests[id].set()
|
||||
return None
|
||||
|
||||
logger.debug(f"Wrote body: {response.code}")
|
||||
with open(fn, "wb") as f:
|
||||
f.write(response.body)
|
||||
self.pendingRequests[id].set()
|
||||
print(type(fn), fn)
|
||||
return fn
|
||||
|
|
@ -7,6 +7,7 @@ voice:
|
|||
port: 4444
|
||||
chunk: 2972
|
||||
google_credentials: "../test_googlespeech/My First Project-0c7833e0d5fa.json"
|
||||
token: "oauth_SOMETHING"
|
||||
hugveys: 25
|
||||
languages:
|
||||
- code: en-GB
|
||||
|
|
|
|||
|
|
@ -264,6 +264,11 @@ class Graph {
|
|||
// used eg. after a condition creation.
|
||||
this.showMsg( this.selectedMsg );
|
||||
}
|
||||
|
||||
getAudioUrlForMsg(msg) {
|
||||
let isVariable = msg['text'].includes('$') ? '1' : '0';
|
||||
return `http://localhost:8888/voice?text=${encodeURIComponent(msg['text'])}&variable=${isVariable}&filename=0`;
|
||||
}
|
||||
|
||||
showMsg( msg ) {
|
||||
let msgEl = document.getElementById( 'msg' );
|
||||
|
|
@ -286,20 +291,19 @@ class Graph {
|
|||
if ( msg['start'] == true ) {
|
||||
startAttributes['checked'] = 'checked';
|
||||
}
|
||||
|
||||
let audioSrcEl = crel('source', {'src': this.getAudioUrlForMsg(msg)});
|
||||
let audioSpan = crel(
|
||||
'span',
|
||||
{
|
||||
'title': msg['audio'] ? msg['audio']['file'] : "",
|
||||
'class': "label-value",
|
||||
},
|
||||
msg['audio'] ? msg['audio']['original_name'] : ""
|
||||
crel(
|
||||
'audio', {'controls': 'controls'},
|
||||
audioSrcEl
|
||||
)
|
||||
);
|
||||
if(msg['audio']) {
|
||||
audioSpan.appendChild(crel(
|
||||
'audio', {'controls': 'controls'},
|
||||
crel('source', {'src':msg['audio']['file']})
|
||||
));
|
||||
}
|
||||
let msgInfoEl = crel( 'div', { 'class': 'msg__info' },
|
||||
crel('div', {
|
||||
'class':'btn btn--delete',
|
||||
|
|
@ -319,7 +323,10 @@ class Graph {
|
|||
'name': msg['@id'] + '-text',
|
||||
'value': msg['text'],
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
'change': this.getEditEventListener(function(){
|
||||
audioSrcEl.src = panopticon.graph.getAudioUrlForMsg(msg);
|
||||
audioSrcEl.parentElement.load();
|
||||
})
|
||||
}
|
||||
} )
|
||||
),
|
||||
|
|
@ -855,7 +862,7 @@ class Graph {
|
|||
* Use wrapper method, because for event handlers 'this' will refer to
|
||||
* the input object
|
||||
*/
|
||||
getEditEventListener() {
|
||||
getEditEventListener(callback) {
|
||||
let graph = this;
|
||||
let el = function( e ) {
|
||||
console.info("Changed", e);
|
||||
|
|
@ -880,6 +887,10 @@ class Graph {
|
|||
// node[field] = e.srcElement.value;
|
||||
|
||||
graph.build();
|
||||
|
||||
if(typeof callback !== 'undefined'){
|
||||
callback();
|
||||
}
|
||||
}
|
||||
return el;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue