From eeed9e31616a840c3345d4657c0ae592c5550bba Mon Sep 17 00:00:00 2001 From: Ruben van de Ven Date: Wed, 24 Apr 2019 16:09:41 +0200 Subject: [PATCH] Timeout diversion --- hugvey/client.py | 7 +-- hugvey/story.py | 101 +++++++++++++++++++++++++++++++----- www/js/hugvey_console.js | 107 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 199 insertions(+), 16 deletions(-) diff --git a/hugvey/client.py b/hugvey/client.py index efb79e3..0aa6187 100644 --- a/hugvey/client.py +++ b/hugvey/client.py @@ -262,6 +262,7 @@ class CommandHandler(object): return logger.info("Received {}".format(cmd)) + if cmd['action'] == 'show_yourself': self.showMyself() if cmd['action'] == 'prepare': @@ -269,7 +270,7 @@ class CommandHandler(object): if cmd['action'] == 'play': self.cmdPlay(cmd) if cmd['action'] == 'stop': - self.cmdPlay(cmd, cmd['id']) + self.cmdStop(cmd['id']) def cmdPlay(self, cmd): self.muteMic = True @@ -317,7 +318,7 @@ class CommandHandler(object): 'msgId': msgId }) out, err = self.playPopen.communicate() - returnCode = self.playPopen.returncode + returnCode = self.playPopen.returncode if self.playPopen else 0 logger.debug('finished') self.playPopen = None @@ -353,7 +354,7 @@ class CommandHandler(object): return # prevent a lock of the story, no repeat or anything for now - logger.warning("Interrupting playback after timeout") + logger.critical("Interrupting playback after timeout") self.playPopen.terminate() def cmdStop(self, msgId): diff --git a/hugvey/story.py b/hugvey/story.py index c372ad7..507adab 100644 --- a/hugvey/story.py +++ b/hugvey/story.py @@ -46,6 +46,7 @@ class Message(object): self.audioFile= None self.filenameFetchLock = asyncio.Lock() self.interruptCount = 0 + self.timeoutDiversionCount = 0 self.afterrunTime = 0. # the time after this message to allow for interrupts self.finishTime = None # message can be finished without finished utterance (with instant replycontains) self.params = {} @@ -445,9 +446,9 @@ class Diversion(object): else: self.regex = None -# if type == 'timeout': -# self.method = self._divergeIfNoResponse -# self.finaliseMethod = self._returnAfterNoResponse + if type == 'timeout': + self.method = self._divergeIfTimeout + self.finaliseMethod = self._returnAfterTimeout if type == 'repeat': self.method = self._divergeIfRepeatRequest self.regex = re.compile(self.params['regex']) @@ -492,7 +493,7 @@ class Diversion(object): Participant doesn't speak for x consecutive replies (has had timeout) """ ':type story: Story' - if story.currentDiversion: + if story.currentDiversion or not msgFrom or not msgTo: return False if story.stats['diversions']['no_response'] + 1 == self.params['timesOccured'] and story.stats['consecutiveSilentTimeouts'] >= int(self.params['consecutiveSilences']): @@ -521,7 +522,7 @@ class Diversion(object): Participant doesn't speak for x consecutive replies (has had timeout) """ ':type story: Story' - if story.currentDiversion: + if story.currentDiversion or not msgFrom or not msgTo: # don't do nested diversions # if we remove this, don't forget to double check 'returnMessage' return False @@ -559,7 +560,8 @@ class Diversion(object): """ Participant asks if message can be repeated. """ - + if not msgFrom or not msgTo: + return # TODO: how to handle this now we sometimes use different timings. # Perhaps set isFinished when matching condition. @@ -574,6 +576,67 @@ class Diversion(object): story.stats['diversions']['repeat'] += 1 await story.setCurrentMessage(msgFrom) return True + + async def _divergeIfTimeout(self, story, msgFrom, msgTo): + """ + (1) last spoken at all + (2) or duration for this last reply only + """ + if msgFrom or msgTo: + # not applicable a direction has been chosen + return + + interval = float(self.params['interval']) + if not self.params['fromLastMessage']: + # (1) last spoken at all + if story.stats['diversions']['timeout_total'] + 1 != self.params['timesOccured']: + return + + timeSince = story.timer.getElapsed('last_speech') if story.timer.hasMark('last_speech') else story.timer.getElapsed('start') + if story.timer.hasMark('last_diversion_timeout') and story.timer.getElapsed('last_diversion_timeout') > timeSince: + timeSince = story.timer.getElapsed('last_diversion_timeout') + if timeSince < interval: + return + + story.stats['diversions']['timeout_total'] += 1 + else: + if story.currentMessage is None: + return + +# if story.currentMessage.timeoutDiversionCount + 1 + if story.stats['diversions']['timeout_last'] + 1 != self.params['timesOccured']: + return + + if story.lastMsgFinishTime is None or story.currentReply is not None: + # still playing back + # or somebody has spoken already (timeout only works on silences) + return + + if time.time() - story.lastMsgFinishTime < interval: + return + + story.currentMessage.timeoutDiversionCount += 1 + story.stats['diversions']['timeout_last'] += 1 + + # if we're still here, there's a match! + story.logger.info(f"Diverge: Timeout {self.id}") + story.stats['diversions']['timeout'] += 1 + + msg = story.get(self.params['msgId']) + if msg is None: + story.logger.critical(f"Not a valid message id for diversion: {self.params['msgId']}") + return + + self.returnMessage = story.currentMessage + await story.setCurrentMessage(msg) + story.currentDiversion = self + story.timer.setMark('last_diversion_timeout') + return True + + async def _returnAfterTimeout(self, story): + story.logger.info(f"Finalise diversion: {self.id}") + if self.params['returnAfterStrand']: + await story.setCurrentMessage(self.returnMessage) storyClasses = { @@ -623,7 +686,9 @@ class Stopwatch(object): def setMark(self, name): self.marks[name] = time.time() - + + def hasMark(self, name): + return name in self.marks def clearMark(self, name): if name in self.marks: @@ -764,7 +829,9 @@ class Story(object): 'no_response': 0, 'repeat': 0, 'reply_contains': 0, - 'timeout': 0 + 'timeout': 0, + 'timeout_total': 0, + 'timeout_last': 0 } } @@ -860,6 +927,7 @@ class Story(object): # messages that come in, in the case google is faster than our playbackFinish event. # (if this setup doesn't work, try to test on self.lastMsgFinish time anyway) # it keeps tricky with all these run conditions + self.logger.info("ignore speech while playing message") continue # message is still playing: @@ -882,6 +950,7 @@ class Story(object): utterance = self.currentReply.getActiveUtterance(now) utterance.setText(e['transcript'], now) self.hugvey.eventLogger.info("speaking: content {} \"{}\"".format(id(utterance), e['transcript'])) + self.timer.setMark('last_speech') if e['is_final']: utterance.setFinished(self.timer.getElapsed()) @@ -893,6 +962,7 @@ class Story(object): async def _processDirections(self, directions): ':type directions: list(Direction)' + chosenDirection = None for direction in directions: for condition in direction.conditions: if condition.isMet(self): @@ -904,14 +974,21 @@ class Story(object): self.addToLog(condition) self.addToLog(direction) self.currentMessage.setFinished(self.timer.getElapsed()) - isDiverging = await self._processDiversions(direction.msgFrom, direction.msgTo) - if not isDiverging: - await self.setCurrentMessage(direction.msgTo) - return direction + chosenDirection = direction + + isDiverging = await self._processDiversions( + chosenDirection.msgFrom if chosenDirection else None, + chosenDirection.msgTo if chosenDirection else None) + if not isDiverging and chosenDirection: + await self.setCurrentMessage(chosenDirection.msgTo) + + return chosenDirection async def _processDiversions(self, msgFrom, msgTo) -> bool: """ Process the diversions on stack. If diverging, return True, else False + msgFrom and msgTo contain the source and target of a headed direction if given + Else, they are None """ diverge = False for diversion in self.diversions: diff --git a/www/js/hugvey_console.js b/www/js/hugvey_console.js index be4b2f5..ce8507f 100644 --- a/www/js/hugvey_console.js +++ b/www/js/hugvey_console.js @@ -445,6 +445,13 @@ class Graph { div['params']['returnAfterStrand'] = true; div['params']['msgId'] = ""; } + else if(type == 'timeout') { + div['params']['interval'] = 20; + div['params']['timesOccured'] = 0; + div['params']['fromLastMessage'] = false; + div['params']['returnAfterStrand'] = true; + div['params']['msgId'] = ""; + } else if(type == 'repeat') { div['params']['regex'] = "can you repeat that\\?"; } else { @@ -603,6 +610,91 @@ class Graph { ) )); } + if(div['type'] == 'timeout') { + let returnAttrs = { + 'type': 'checkbox', + 'on': { + 'change': (e) => div['params']['returnAfterStrand'] = e.target.checked + } + } + if(div['params']['returnAfterStrand']) { + returnAttrs['checked'] = 'checked'; + } + + let totalOrLocalAttrs = { + 'type': 'checkbox', + 'on': { + 'change': (e) => div['params']['fromLastMessage'] = e.target.checked + } + } + if(div['params']['fromLastMessage']) { + totalOrLocalAttrs['checked'] = 'checked'; + } + + let msgOptions = [crel('option',"")]; + let starts = this.messages.filter( m => m.hasOwnProperty('start') && m['start'] == true); + for(let startMsg of starts) { + let optionParams = {}; + if(div['params']['msgId'] == startMsg['@id']) { + optionParams['selected'] = 'selected'; + } + msgOptions.push(crel('option', optionParams , startMsg['@id'])); + } + + divsTimeouts.push(crel( + 'div', { + 'class': 'diversion', + 'on': { + 'mouseover': function(e) { + if(div['params']['msgId']) + document.getElementById(div['params']['msgId']).classList.add('selectedMsg'); + }, + 'mouseout': function(e) { + if(div['params']['msgId']) + document.getElementById(div['params']['msgId']).classList.remove('selectedMsg'); + } + } + }, + crel('h3', div['@id']), + crel( + 'div', { + 'class':'btn btn--delete', + 'on': { + 'click': (e) => this.deleteDiversion(div) + } + }, 'Delete diversion'), + crel('label', 'For last message only', + crel('input', totalOrLocalAttrs) + ), + crel('label', 'Seconds of silence', + crel('input', { + 'type': 'number', + 'value': div['params']['interval'], + 'precision': .1, + 'on': { + 'change': (e) => div['params']['interval'] = parseFloat(e.target.value) + } + }) + ), + crel('label', 'On n-th instance', + crel('input', { + 'type': 'number', + 'value': div['params']['timesOccured'], + 'on': { + 'change': (e) => div['params']['timesOccured'] = parseInt(e.target.value) + } + }) + ), + crel('label', 'Return to point of departure afterwards', + crel('input', returnAttrs) + ), + crel('label', 'Go to (start message)', + crel('select', {'on': { + 'change': (e) => div['params']['msgId'] = e.target.value + }}, ...msgOptions) + ) + )); + } if(div['type'] == 'repeat'){ divsRepeat.push(crel( 'div', {'class': 'diversion'}, @@ -627,7 +719,7 @@ class Graph { } } - console.log(divsReplyContains, divsNoResponse, divsRepeat); + console.log(divsReplyContains, divsNoResponse, divsRepeat, divsTimeouts); let divEl = crel( 'div', @@ -673,6 +765,19 @@ class Graph { }, 'New case for repeat' ) + ), + crel('div', + crel('h2', 'Timeouts'), + ...divsTimeouts, + crel('div', + { + 'class': 'btn', + 'on': { + 'click': (e) => this.createDiversion('timeout') + } + }, + 'New case for timeout' + ) ) );