From 554f6a2cc39c28c7e93b4cbd71e25663fa87841a Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Fri, 12 Apr 2019 12:38:00 +0200 Subject: [PATCH] Mute speech events before playing back audio --- hugvey/client.py | 17 +++++++++++++---- hugvey/speech/google.py | 3 ++- hugvey/story.py | 28 +++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/hugvey/client.py b/hugvey/client.py index 3b5a6ef..fb3ce96 100644 --- a/hugvey/client.py +++ b/hugvey/client.py @@ -158,9 +158,9 @@ class VoiceServer(object): in_data, 2, 1, self.input_rate, self.target_rate, self.laststate) try: - if self.hugvey.cmd_server.playPopen is not None: + if self.hugvey.cmd_server.muteMic: logger.debug('block recording {}' .format( - self.hugvey.cmd_server.playPopen)) + self.hugvey.cmd_server.muteMic)) # if recording is blocked, store the latest n frames. So we can release # this buffer to the mic stream @@ -175,7 +175,7 @@ class VoiceServer(object): logger.info('used buffer, len now {}'.format(len(f))) - self.loop.call_soon_threadsafe(self.voice_socket.send, f) + self.loop.call_soon_threadsafe(self.voice_socket.send, f) except Exception as e: logger.warn("Error sending to {}".format(e)) pass @@ -256,6 +256,7 @@ class CommandHandler(object): self.hugvey_id = hugvey_id self.cmd_address = cmd_address self.publish_address = publish_address + self.muteMic = False self.playPopen = None self.file_address = file_address self.playingMsgId = None @@ -272,12 +273,16 @@ class CommandHandler(object): logger.info("Received {}".format(cmd)) if cmd['action'] == 'show_yourself': self.showMyself() + if cmd['action'] == 'prepare': + self.muteMic = True if cmd['action'] == 'play': self.cmdPlay(cmd) if cmd['action'] == 'stop': self.cmdPlay(cmd, cmd['id']) def cmdPlay(self, cmd): + self.muteMic = True + msgId = cmd['id'] pitch = cmd['pitch'] if 'pitch' in cmd else 50 file = cmd['file'] if 'file' in cmd else None @@ -308,7 +313,10 @@ class CommandHandler(object): logger.debug(playCmd) self.playPopen = subprocess.Popen( playCmd, stdout=subprocess.PIPE, env=environment_vars) - + self.sendMessage({ + 'event': 'playbackStart', + 'msgId': msgId + }) returnCode = self.playPopen.wait() logger.debug('finished') self.playPopen = None @@ -326,6 +334,7 @@ class CommandHandler(object): "Finished playback. Return code: {}".format(returnCode)) self.playingMsgId = None + self.muteMic = False self.sendMessage({ 'event': 'playbackFinish', 'msgId': msgId diff --git a/hugvey/speech/google.py b/hugvey/speech/google.py index 62dede9..f976fee 100644 --- a/hugvey/speech/google.py +++ b/hugvey/speech/google.py @@ -190,7 +190,8 @@ class GoogleVoiceClient(object): self.logger.info("Pause muted stream!") self.pause() return - + + self.subsequentMutedFrames = 0 # self.logger.debug("We have mic!") if not self.isRunning.is_set(): self.logger.info("Resume voice") diff --git a/hugvey/story.py b/hugvey/story.py index 64c5451..c0fa675 100644 --- a/hugvey/story.py +++ b/hugvey/story.py @@ -745,7 +745,14 @@ class Story(object): # that is, until we have a 'reset' or 'start' event. # reinitiate current message await self.setCurrentMessage(self.currentMessage) - + + if e['event'] == "playbackStart": + if e['msgId'] != self.currentMessage.id: + continue + self.lastMsgStartTime = self.timer.getElapsed() + self.logger.debug("Start playback") + + if e['event'] == "playbackFinish": if e['msgId'] == self.currentMessage.id: #TODO: migrate value to Messagage instead of Story @@ -771,6 +778,16 @@ class Story(object): # participants speaks, reset counter self.stats['consecutiveSilentTimeouts'] = 0 +# if self.currentMessage and not self.lastMsgStartTime: + if self.currentMessage and not self.lastMsgFinishTime: + # Ignore incoming speech events until we receive a 'playbackStart' event. + # After that moment the mic will be muted, so nothing should come in _anyway_ + # unless google is really slow on us. But by taking the start time we don't ignore + # messages that come in, in the case google is faster than our playbackFinish event. + # (if this setup doesn't work, try to test on self.lastMsgFinish time anyway) + # it keeps tricky with all these run conditions + continue + # message is still playing: if self.currentMessage and not self.lastMsgFinishTime and self.previousReply and self.previousReply.forMessage.interruptCount < 4: timeDiff = self.timer.getElapsed() - self.previousReply.forMessage.getFinishedTime() @@ -883,10 +900,18 @@ class Story(object): self.currentMessage = message self.lastMsgTime = time.time() self.lastMsgFinishTime = None # to be filled in by the event + self.lastMsgStartTime = None # to be filled in by the event # if not reset: self.previousReply = self.currentReply # we can use this for interrptions self.currentReply = useReply #self.currentMessage.reply + + # send command to already mute mic + self.hugvey.sendCommand({ + 'action': 'prepare', + 'id': message.id + }) + # else: # # if we press 'save & play', it should not remember it's last reply to that msg # self.previousReply = self.currentReply # we can use this for interrptions @@ -901,6 +926,7 @@ class Story(object): # TODO: prep events & timer etc. fn = await message.getAudioFilePath() +# self.hugvey.google.pause() # pause STT to avoid text events while decision is made self.hugvey.sendCommand({ 'action': 'play', 'file': fn,