diff --git a/client_config.yml b/client_config.yml index ec74ec9..685182a 100644 --- a/client_config.yml +++ b/client_config.yml @@ -6,3 +6,7 @@ voice: target_rate: 16000 port: 4444 input_name: null + file_address: "http://192.168.178.185:8888" + play_device: null + play_volume: 80 + diff --git a/hugvey/central_command.py b/hugvey/central_command.py index 13f432f..15fa758 100644 --- a/hugvey/central_command.py +++ b/hugvey/central_command.py @@ -74,7 +74,6 @@ class CentralCommand(object): self.languageFiles[lang['code']] = lang['file'] with open(lang_filename, 'r') as fp: self.languages[lang['code']] = json.load(fp) - print(self.languages) self.panopticon = Panopticon(self, self.config) @@ -85,9 +84,9 @@ class CentralCommand(object): return status hv = self.hugveys[hv_id] - status['status'] = 'running' if hv.isRunning.is_set() else 'paused' + status['status'] = hv.getStatus() status['language'] = hv.language_code - status['msg'] = hv.story.currentMessage.id + status['msg'] = hv.story.currentMessage.id if hv.story.currentMessage else None status['counts'] = hv.story.getStoryCounts() status['finished'] = hv.story.isFinished() @@ -168,16 +167,12 @@ class CentralCommand(object): async with self.hugveyLock: # lock to prevent duplicates on creation if not hugvey_id in self.hugveys: logger.info(f'Instantiate hugvey #{hugvey_id}') - print('a') h = HugveyState(hugvey_id, self) - print('a') h.config(msg['host'], msg['ip']) - print('b') self.hugveys[hugvey_id] = h thread = threading.Thread( target=h.start, name=f"hugvey#{hugvey_id}") thread.start() - print('c') else: logger.info(f'Reconfigure hugvey #{hugvey_id}') # (re)configure exisitng hugveys @@ -210,7 +205,8 @@ class CentralCommand(object): logger.debug("Message contains: {}".format(msg)) continue else: - await self.hugveys[hugvey_id].eventQueue.put(msg) + self.hugveys[hugvey_id].queueEvent(msg) +# await self.hugveys[hugvey_id].eventQueue.put(msg) except Exception as e: logger.critical(f"Exception while running event loop:") logger.exception(e) @@ -242,6 +238,10 @@ class HugveyState(object): """Represents the state of a Hugvey client on the server. Manages server connections & voice parsing etc. """ + + STATE_PAUSE = "paused" + STATE_GONE = "gone" + STATE_RUNNING = "running" def __init__(self, id: int, command: CentralCommand): @@ -255,7 +255,16 @@ class HugveyState(object): self.language_code = 'en-GB' self.story = Story(self) self.story.setStoryData(self.command.languages[self.language_code]) + self.streamer = None + self.status = self.STATE_PAUSE + self.google = None + self.notShuttingDown = True # TODO: allow shutdown of object + def getStatus(self): + if self.story.isFinished(): + return "finished" + return self.status + def config(self, hostname, ip): self.ip = ip self.hostname = hostname @@ -279,13 +288,17 @@ class HugveyState(object): """ Start the tasks """ + self.isRunning.set() + self.status = self.STATE_RUNNING + tasks = asyncio.gather( self.catchException(self.processAudio()), self.catchException(self.handleEvents()), self.catchException(self.playStory()), loop=self.loop) +# self.pause() self.loop.run_until_complete(tasks) - self.isRunning.set() + async def catchException(self, awaitable): try: @@ -336,46 +349,73 @@ class HugveyState(object): self.story.setStoryData(self.command.languages[language_code]) def pause(self): - self.google.pause() + self.logger.info('Pause') + if self.google: + self.google.pause() self.story.pause() self.isRunning.clear() + self.status = self.STATE_PAUSE def resume(self): - self.google.resume() + self.logger.info('Resume') + if self.google: + self.google.resume() self.story.resume() self.isRunning.set() + self.status = self.STATE_RUNNING def restart(self): + self.logger.info('Restart') self.story.reset() self.resume() self.isRunning.set() + + def gone(self): + '''Status to 'gone' as in, shutdown/crashed/whatever + ''' + self.pause() + self.logger.info('Gone') + self.status = self.STATE_GONE + async def playStory(self): await self.story.start() + + def getStreamer(self): + if not self.streamer: + self.streamer = AudioStreamer( + self.command.config['voice']['chunk'], + self.ip, + int(self.command.config['voice']['port'])) + + if self.command.debug: + self.logger.warn("Debug on: Connecting Audio player") + self.player = Player( + self.command.config['voice']['src_rate'], self.command.config['voice']['out_rate']) + self.streamer.addConsumer(self.player) + + self.logger.info("Start Speech") + self.google = GoogleVoiceClient( + hugvey=self, + src_rate=self.command.config['voice']['src_rate'], + credential_file=self.command.config['voice']['google_credentials'], + language_code=self.language_code + ) + self.streamer.addConsumer(self.google) + return self.streamer async def processAudio(self): ''' Start the audio streamer service ''' + self.logger.info("Start audio stream") - streamer = AudioStreamer( - self.command.config['voice']['chunk'], - self.ip, - int(self.command.config['voice']['port'])) - if self.command.debug: - self.logger.warn("Debug on: Connecting Audio player") - self.player = Player( - self.command.config['voice']['src_rate'], self.command.config['voice']['out_rate']) - streamer.addConsumer(self.player) - - self.logger.info("Start Speech") - self.google = GoogleVoiceClient( - hugvey=self, - src_rate=self.command.config['voice']['src_rate'], - credential_file=self.command.config['voice']['google_credentials'], - language_code=self.language_code - ) - streamer.addConsumer(self.google) - - await streamer.run() + while self.notShuttingDown: +# self.isRunning.wait() + + self.logger.info("Start audio stream") + await self.getStreamer().run() + self.logger.warn("stream has left the building") + # if we end up here, the streamer finished, probably meaning hte hugvey shutdown + self.gone() diff --git a/hugvey/client.py b/hugvey/client.py index 557858f..8ba7f05 100644 --- a/hugvey/client.py +++ b/hugvey/client.py @@ -7,6 +7,7 @@ import socket import threading import time import yaml +import alsaaudio import zmq from zmq.asyncio import Context @@ -112,13 +113,14 @@ class VoiceServer(object): r = await future class CommandHandler(object): - def __init__(self, hugvey_id, cmd_address = "tcp://127.0.0.1:5555", publish_address = "tcp://0.0.0.0:5555"): + def __init__(self, hugvey_id, cmd_address, publish_address, file_address): self.eventQueue = [] self.ctx = Context.instance() self.hugvey_id = hugvey_id self.cmd_address = cmd_address self.publish_address = publish_address self.playPopen = None + self.file_address = file_address # self.showMyself() # queue message for connection request def handle(self, cmd): @@ -132,20 +134,34 @@ class CommandHandler(object): if cmd['action'] == 'show_yourself': self.showMyself() if cmd['action'] == 'play': - self.cmdPlay(cmd['id'], cmd['msg']) + self.cmdPlay(cmd) - def cmdPlay(self, msgId, msgText, pitch=50): - logger.info("Play: {}".format(msgText)) - self.playPopen = subprocess.Popen(['espeak', '-p','{0}'.format(pitch), msgText], stdout=subprocess.PIPE) - returnCode = self.playPopen.wait() - self.playPopen = None + def cmdPlay(self, cmd): + msgId= cmd['id'] + pitch = cmd['pitch'] if 'pitch' in cmd else 50 + file = cmd['file'] if 'file' in cmd else None + text = cmd['msg'] if 'msg' in cmd else None - if returnCode: - logger.warn("Had returncode on play: {}".format(returnCode)) + if file is None and text is None: + logger.critical("No file nor text given: {}".format(cmd)) else: - logger.debug("Finished playback. Return code: {}".format(returnCode)) - + if file is not None: + logger.info("Play: {}".format(file)) + file = self.file_address + "/" + file + self.playPopen = subprocess.Popen(['play', file], stdout=subprocess.PIPE) + returnCode = self.playPopen.wait() + self.playPopen = None + else: + logger.info("Speak: {}".format(text)) + self.playPopen = subprocess.Popen(['espeak', '-p','{0}'.format(pitch), text], stdout=subprocess.PIPE) + returnCode = self.playPopen.wait() + self.playPopen = None + + if returnCode: + logger.warn("Had returncode on play: {}".format(returnCode)) + else: + logger.debug("Finished playback. Return code: {}".format(returnCode)) self.sendMessage({ 'event': 'playbackFinish', @@ -237,6 +253,9 @@ class Hugvey(object): def start(self): loop = asyncio.get_event_loop() + if self.config['voice']['play_device']: + alsaaudio.Mixer(self.config['voice']['play_device']).setvolume(self.config['voice']['play_volume']) + self.voice_server = VoiceServer( loop = loop, voice_port = int(self.config['voice']['port']), @@ -248,6 +267,7 @@ class Hugvey(object): hugvey_id = self.id, cmd_address = self.config['events']['cmd_address'], publish_address = self.config['events']['publish_address'], + file_address = self.config['voice']['file_address'] ) logger.info('start') # self.voice_server.asyncStart(loop) diff --git a/hugvey/panopticon.py b/hugvey/panopticon.py index 0859238..6a9631c 100644 --- a/hugvey/panopticon.py +++ b/hugvey/panopticon.py @@ -56,6 +56,7 @@ def getWebSocketHandler(central_command): except Exception as e: self.send({'alert': 'Invalid request: {}'.format(e)}) + logger.exception(e) def send(self, message): j = json.dumps(message) @@ -80,13 +81,13 @@ def getWebSocketHandler(central_command): self.send(msg) def msgResume(self, hv_id): - central_command.hugveys[hv_id].eventQueue.put({'event': 'resume'}) + central_command.hugveys[hv_id].eventQueue.put_nowait({'event': 'resume'}) def msgPause(self, hv_id): - central_command.hugveys[hv_id].eventQueue.put({'event': 'pause'}) + central_command.hugveys[hv_id].eventQueue.put_nowait({'event': 'pause'}) def msgRestart(self, hv_id): - central_command.hugveys[hv_id].eventQueue.put({'event': 'restart'}) + central_command.hugveys[hv_id].eventQueue.put_nowait({'event': 'restart'}) return WebSocketHandler diff --git a/hugvey/story.py b/hugvey/story.py index 6f07d75..6d70f76 100644 --- a/hugvey/story.py +++ b/hugvey/story.py @@ -14,11 +14,14 @@ class Message(object): self.text = text self.isStart = False self.reply = None + self.audioFile= None @classmethod def initFromJson(message, data, story): msg = message(data['@id'], data['text']) msg.isStart = data['start'] if 'start' in data else False + if 'audio' in data: + msg.audioFile = data['audio']['file'] return msg def setReply(self, text): @@ -123,9 +126,9 @@ class Direction(object): return direction -class Interruption(object): +class Diversion(object): """ - An Interruption. Used to catch events outside of story flow. + An Diversion. Used to catch events outside of story flow. """ def __init__(self, id): @@ -136,20 +139,20 @@ class Interruption(object): self.conditions.append(condition) @classmethod - def initFromJson(interruptionClass, data, story): - interrupt = interruptionClass(data['@id']) + def initFromJson(diversionClass, data, story): + diversion = diversionClass(data['@id']) if 'conditions' in data: for conditionId in data['conditions']: c = story.get(conditionId) - interrupt.addCondition(c) - return interrupt + diversion.addCondition(c) + return diversion storyClasses = { 'Msg': Message, 'Direction': Direction, 'Condition': Condition, - 'Interruption': Interruption, + 'Diversion': Diversion, } @@ -231,7 +234,7 @@ class Story(object): # return counts return { 'messages': len([e for e in self.log if isinstance(e, Message)]), - 'interruptions': len([e for e in self.log if isinstance(e, Interruption)]) + 'diversions': len([e for e in self.log if isinstance(e, Diversion)]) } def setStoryData(self, story_data): @@ -244,7 +247,7 @@ class Story(object): currentId = self.currentMessage.id if self.currentMessage else None self.elements = {} - self.interruptions = [] + self.diversions = [] self.directionsPerMsg = {} self.startMessage = None # The entrypoint to the graph self.reset() @@ -291,8 +294,8 @@ class Story(object): self.elements[obj.id] = obj - if type(obj) == Interruption: - self.interruptions.append(obj) + if type(obj) == Diversion: + self.diversions.append(obj) if type(obj) == Direction: if obj.msgFrom.id not in self.directionsPerMsg: @@ -399,11 +402,18 @@ class Story(object): message.id, message.text)) self.log.append(message) # TODO: prep events & timer etc. - self.hugvey.sendCommand({ - 'action': 'play', - 'msg': message.text, - 'id': message.id, - }) + if message.audioFile: + self.hugvey.sendCommand({ + 'action': 'play', + 'file': message.audioFile, + 'id': message.id, + }) + else: + self.hugvey.sendCommand({ + 'action': 'play', + 'msg': message.text, + 'id': message.id, + }) logger.debug("Pending directions: ") @@ -426,8 +436,8 @@ class Story(object): await self._renderer() def isFinished(self): - if hasattr(self, 'finish_time'): - return self.finish_time + if hasattr(self, 'finish_time') and self.finish_time: + return time.time() - self.finish_time return False diff --git a/hugvey/voice/streamer.py b/hugvey/voice/streamer.py index 570fd6f..ff6bca5 100644 --- a/hugvey/voice/streamer.py +++ b/hugvey/voice/streamer.py @@ -27,6 +27,7 @@ class AudioStreamer(object): address = "tcp://{}:{}".format(self.address, self.port) self.ctx = Context.instance() self.socket = self.ctx.socket(zmq.SUB) + self.socket.setsockopt(zmq.RCVTIMEO, 4000) # timeout: 8 sec self.socket.subscribe('') # self.socket.setsockopt(zmq.CONFLATE, 1) self.socket.connect(address) @@ -35,13 +36,16 @@ class AudioStreamer(object): logger.info("Attempt connection on {}:{}".format(self.address, self.port)) # s.connect((self.address, self.port)) # - while self.isRunning: - data = await self.socket.recv() -# logger.debug('chunk received') - self.process(data) - - logger.info("Close socket on {}:{}".format(self.address, self.port)) - self.socket.close() + try: + while self.isRunning: + data = await self.socket.recv() + # logger.debug('chunk received') + self.process(data) + except zmq.error.Again as timeout_e: + logger.warn("Timeout of audiostream. Hugvey shutdown?") + finally: + logger.info("Close socket on {}:{}".format(self.address, self.port)) + self.socket.close() def stop(self): self.isRunning = False diff --git a/requirements.txt b/requirements.txt index c40d64f..9f3d7ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ pyzmq pyaudio coloredlogs pyyaml +pyalsaaudio diff --git a/www/css/styles.css b/www/css/styles.css index 90663ae..d8af211 100644 --- a/www/css/styles.css +++ b/www/css/styles.css @@ -19,6 +19,10 @@ body { to { stroke-dashoffset: -1000; } } +img.icon { + height: .9em; + width: .9em; } + #interface { display: flex; flex-direction: row; @@ -40,6 +44,14 @@ body { position: relative; } #status > div#overview { width: 66.66667%; } + #status .counts dd, #status .counts dt { + display: inline-block; + width: 30px; + overflow: hidden; + text-overflow: ellipsis; + margiN: 0; } + #status .counts dd:hover, #status .counts dt:hover { + width: auto; } #status .hugvey { background-image: linear-gradient(to top, #587457, #35a589); color: white; @@ -57,6 +69,27 @@ body { #status .hugvey.hugvey--off { background-image: linear-gradient(to top, #575d74, #3572a5); } #status .hugvey.hugvey--off::after { + content: 'off'; + font-style: italic; + color: gray; + text-align: center; } + #status .hugvey.hugvey--gone { + background-image: linear-gradient(to top, orange, #ce5c00); } + #status .hugvey.hugvey--gone::after { + content: 'disconnected'; + font-style: italic; + color: gray; + text-align: center; } + #status .hugvey.hugvey--paused { + background-image: linear-gradient(to top, #888a85, #555753); } + #status .hugvey.hugvey--paused::after { + content: 'disconnected'; + font-style: italic; + color: gray; + text-align: center; } + #status .hugvey.hugvey--finished { + background-image: linear-gradient(to top, #888a85, #35a589); } + #status .hugvey.hugvey--finished::after { content: 'disconnected'; font-style: italic; color: gray; diff --git a/www/images/icon-diversions.svg b/www/images/icon-diversions.svg new file mode 100644 index 0000000..cb21c6f --- /dev/null +++ b/www/images/icon-diversions.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/images/icon-finished.svg b/www/images/icon-finished.svg new file mode 100644 index 0000000..2074db3 --- /dev/null +++ b/www/images/icon-finished.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/images/icon-interruptions.svg b/www/images/icon-interruptions.svg new file mode 100644 index 0000000..9ab2d38 --- /dev/null +++ b/www/images/icon-interruptions.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/images/icon-laughs.svg b/www/images/icon-laughs.svg new file mode 100644 index 0000000..b852425 --- /dev/null +++ b/www/images/icon-laughs.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/images/icon-messages.svg b/www/images/icon-messages.svg new file mode 100644 index 0000000..de4dc07 --- /dev/null +++ b/www/images/icon-messages.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/images/icon-timeouts.svg b/www/images/icon-timeouts.svg new file mode 100644 index 0000000..ebeece4 --- /dev/null +++ b/www/images/icon-timeouts.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/www/index.html b/www/index.html index 159983e..5df9e66 100644 --- a/www/index.html +++ b/www/index.html @@ -7,6 +7,7 @@ + @@ -18,28 +19,33 @@
Uptime
{{uptime}}
- +
+ :class="[{'hugvey--on': hv.status != 'off'},'hugvey--' + hv.status]">

{{ hv.id }} -

-
- {{ hv.language }} / {{ hv.msg }} -
Finished: {{time_passed(hv, +
{{ hv.status }}
+
+ {{ hv.language }} / {{ hv.msg }} +
{{timer(hv, 'finished')}}
-
-
{{key}}
-
{{c}}
+
+
+ + {{c}} +
-
Pause
-
Resume
+
Pause
+
Resume
+
Restart
diff --git a/www/js/hugvey_console.js b/www/js/hugvey_console.js index 6d5632b..63c30b8 100644 --- a/www/js/hugvey_console.js +++ b/www/js/hugvey_console.js @@ -3,6 +3,7 @@ var panopticon; class Panopticon { constructor() { console.log( "Init panopticon" ); + this.languages = [] this.hugveys = new Vue( { el: "#status", data: { @@ -15,15 +16,27 @@ class Panopticon { console.log( "property!", Date( hugvey[property] * 1000 ) ); return moment( Date( hugvey[property] * 1000 ) ).fromNow(); }, + timer: function(hugvey, property) { + return panopticon.stringToHHMMSS( hugvey[property] ); + }, loadNarrative: function( code, file ) { return panopticon.loadNarrative( code, file ); + }, + pause: function(hv_id) { + return panopticon.pause(hv_id); + }, + resume: function(hv_id) { + return panopticon.resume(hv_id); + }, + restart: function(hv_id) { + return panopticon.restart(hv_id); } } } ); - this.socket = new ReconnectingWebSocket( "ws://localhost:8888/ws", null, { debug: true, reconnectInterval: 3000 } ); + this.socket = new ReconnectingWebSocket( "ws://localhost:8888/ws", null, { debug: false, reconnectInterval: 3000 } ); this.graph = new Graph(); @@ -48,8 +61,10 @@ class Panopticon { switch ( msg['action'] ) { case 'status': + console.debug(msg); this.hugveys.uptime = this.stringToHHMMSS( msg['uptime'] ); this.hugveys.languages = msg['languages']; + this.languages = msg['languages']; this.hugveys.hugveys = msg['hugveys']; break; } @@ -85,8 +100,14 @@ class Panopticon { return hours + ':' + minutes + ':' + seconds; } - loadNarrative( code, file ) { + if(typeof file == 'undefined') { + for (let lang of this.languages) { + if (lang['code'] == code) { + file = lang['file']; + } + } + } let req = new XMLHttpRequest(); let graph = this.graph; req.addEventListener( "load", function( e ) { @@ -128,7 +149,7 @@ class Graph { this.messages = []; // initialise empty array. For the simulation, make sure we keep the same array object this.directions = []; // initialise empty array. For the simulation, make sure we keep the same array object this.conditions = []; // initialise empty array. For the simulation, make sure we keep the same array object - this.interruptions = []; // initialise empty array. For the simulation, make sure we keep the same array object + this.diversions = []; // initialise empty array. For the simulation, make sure we keep the same array object let graph = this; this.controlDown = false; @@ -259,6 +280,9 @@ class Graph { panopticon.graph.saveJson(msg['@id'], e.target, function(e2){ console.log(e); audioSpan.innerHTML = e.target.files[0].name + "*"; + // reload graph: + console.log('reload', panopticon.graph.language_code); + panopticon.loadNarrative(panopticon.graph.language_code); }); // console.log(this,e); } @@ -596,7 +620,7 @@ class Graph { getJsonString() { // recreate array to have the right order of items. this.data = [...this.messages, ...this.conditions, - ...this.directions, ...this.interruptions] + ...this.directions, ...this.diversions] let d = []; let toRemove = ['sourceX', 'sourceY', 'targetX', 'targetY', 'x', 'y', 'vx', 'vy'] for ( let node of this.data ) { @@ -673,7 +697,7 @@ class Graph { this.messages = this.data.filter(( node ) => node['@type'] == 'Msg' ); this.directions = this.data.filter(( node ) => node['@type'] == 'Direction' ); this.conditions = this.data.filter(( node ) => node['@type'] == 'Condition' ); - this.interruptions = this.data.filter(( node ) => node['@type'] == 'Interruption' ); + this.diversions = this.data.filter(( node ) => node['@type'] == 'Diversion' ); document.getElementById('current_lang').innerHTML = ""; document.getElementById('current_lang').appendChild(crel('span', { diff --git a/www/narrative_builder.html b/www/narrative_builder.html deleted file mode 100644 index c5cf754..0000000 --- a/www/narrative_builder.html +++ /dev/null @@ -1,832 +0,0 @@ - - - - - Pillow Talk - Narrative Builder - - - - - -
-

Hugvey

-

Select a narrative json file

- -
- - -
-
-
-
-
Save JSON
-
New Message
-
- - - - - - - - - - diff --git a/www/scss/styles.scss b/www/scss/styles.scss index 203bfef..5967404 100644 --- a/www/scss/styles.scss +++ b/www/scss/styles.scss @@ -28,6 +28,11 @@ body{ } } +img.icon{ + height: .9em; + width: .9em; +} + #interface{ display: flex; flex-direction: row; @@ -58,6 +63,19 @@ body{ } } + .counts{ + dd, dt{ + display: inline-block; + width: 30px; + overflow:hidden; + text-overflow: ellipsis; + margiN: 0; + &:hover{ + width: auto; + } + } + } + .hugvey{ background-image: linear-gradient(to top, #587457, #35a589); color:white; @@ -81,6 +99,39 @@ body{ &.hugvey--off{ background-image: linear-gradient(to top, #575d74, #3572a5); + &::after{ + content: 'off'; + font-style: italic; + color: gray; + text-align:center; +// font-size: 30pt; + } + } + + &.hugvey--gone{ + background-image: linear-gradient(to top, orange, rgb(206, 92, 0)); + &::after{ + content: 'disconnected'; + font-style: italic; + color: gray; + text-align:center; +// font-size: 30pt; + } + } + + &.hugvey--paused{ + background-image: linear-gradient(to top, rgb(136, 138, 133), rgb(85, 87, 83)); + &::after{ + content: 'disconnected'; + font-style: italic; + color: gray; + text-align:center; +// font-size: 30pt; + } + } + + &.hugvey--finished{ + background-image: linear-gradient(to top, rgb(136, 138, 133), #35a589); &::after{ content: 'disconnected'; font-style: italic;