diff --git a/hugvey/communication.py b/hugvey/communication.py index a0b51b8..e6776da 100644 --- a/hugvey/communication.py +++ b/hugvey/communication.py @@ -3,6 +3,9 @@ import logging logger = logging.getLogger("communication") +# hyper verbose log level. Have it here, becase it needs to be _somewhere_ +LOG_BS = 5 + def getTopic(hugvey_id): return "hv{}".format(hugvey_id) diff --git a/hugvey/story.py b/hugvey/story.py index 04398ac..2098ba9 100644 --- a/hugvey/story.py +++ b/hugvey/story.py @@ -3,50 +3,126 @@ import time import logging import re import asyncio +from .communication import LOG_BS logger = logging.getLogger("narrative") +class Utterance(object): + """Part of a reply""" + def __init__(self, startTime): + self.startTime = startTime + self.endTime = None + self.text = "" + + def setText(self, text): + self.text = text + + def setFinished(self, endTime): + self.endTime = endTime + + def isFinished(self): + return self.endTime is not None + + class Message(object): def __init__(self, id, text): self.id = id self.text = text self.isStart = False self.reply = None - self.replyTime = None +# self.replyTime = None self.audioFile= None + self.interruptCount = 0 + self.afterrunTime = 0. # the time after this message to allow for interrupts + self.finishTime = None # message can be finished without finished utterance (with instant replycontains) @classmethod def initFromJson(message, data, story): msg = message(data['@id'], data['text']) msg.isStart = data['start'] if 'start' in data else False + msg.afterrunTime = data['afterrun'] if 'afterrun' in data else 0. if 'audio' in data: msg.audioFile = data['audio']['file'] return msg - - def setReply(self, text, replyTime): - self.reply = text - self.replyTime = replyTime + + def setReply(self, reply): + self.reply = reply def hasReply(self): return self.reply is not None def getReply(self): - if self.reply is None: + if not self.hasReply(): raise Exception( "Getting reply while there is none! {0}".format(self.id)) return self.reply + + def isFinished(self): + return self.finishTime is not None + + def setFinished(self, currentTime): + self.finishTime = currentTime + + def getFinishedTime(self): + return self.finishTime def getLogSummary(self): return { 'id': self.id, - 'time': self.replyTime, - 'replyText': self.reply + 'time': None if self.reply is None else [u.startTime for u in self.reply.utterances], + 'replyText': None if self.reply is None else [u.text for u in self.reply.utterances] } +class Reply(object): + def __init__(self, message: Message): + self.forMessage = None + self.utterances = [] + self.setForMessage(message) + + def setForMessage(self, message: Message): + self.forMessage = message + message.setReply(self) + + def getLastUtterance(self) -> Utterance: + if not self.hasUtterances(): + return None + return self.utterances[-1] + + def getFirstUtterance(self) -> Utterance: + if not self.hasUtterances(): + return None + return self.utterances[0] + + def hasUtterances(self) -> bool: + return len(self.utterances) > 0 + + def addUtterance(self, utterance: Utterance): + self.utterances.append(utterance) + + def getText(self) -> str: + return ". ".join([u.text for u in self.utterances]) + + def getActiveUtterance(self, currentTime) -> Utterance: + """ + If no utterance is active, create a new one. Otherwise return non-finished utterance for update + """ + if len(self.utterances) < 1 or self.getLastUtterance().isFinished(): + u = Utterance(currentTime) + self.addUtterance(u) + else: + u = self.getLastUtterance() + return u + + def isSpeaking(self): + u = self.getLastUtterance() + if u is not None and not u.isFinished(): + return True + return False + class Condition(object): """ A condition, basic conditions are built in, custom condition can be given by @@ -86,28 +162,57 @@ class Condition(object): return now - story.lastMsgFinishTime >= float(self.vars['seconds']) - def _hasMetReplyContains(self, story): - if not story.currentMessage.hasReply(): + def _hasMetReplyContains(self, story) -> bool: + """ + Check the reply for specific characteristics: + - regex: regular expression. If empy, just way for isFinished() + - delays: an array of [{'minReplyDuration', 'waitTime'},...] + - minReplyDuration: the nr of seconds the reply should take. Preferably have one with 0 + - waitTime: the time to wait after isFinished() before continuing + """ + r = story.currentReply # make sure we keep working with the same object + if not r or not r.hasUtterances(): return False - if 'regex' in self.vars: + if 'regex' in self.vars and len(self.vars['regex']): if 'regexCompiled' not in self.vars: # Compile once, as we probably run it more than once self.vars['regexCompiled'] = re.compile(self.vars['regex']) - result = re.match(self.vars['regexCompiled'], story.currentMessage.getReply()) + + t = story.currentReply.getText().lower() + logger.log(LOG_BS, 'attempt regex: {} on {}'.format(self.vars['regex'], t)) + result = self.vars['regexCompiled'].search(t) if result is None: + #if there is something to match, but not found, it's never ok return False + logger.debug('Got match on {}'.format(self.vars['regex'])) results = result.groupdict() for captureGroup in results: story.variables[captureGroup] = results[captureGroup] - logger.critical("Regex not implemented yet") -# return True + + # TODO: implement 'instant match' -> don't wait for isFinished() + + if r.isSpeaking(): + logger.log(LOG_BS, "is speaking") return False - - if 'contains' in self.vars: - if self.vars['contains'] == '*': - return True - return self.vars['contains'] in story.currentMessage.getReply() + +# print(self.vars) + # either there's a match, or nothing to match at all + if 'delays' in self.vars: + replyDuration = story.timer.getElapsed() - r.getFirstUtterance().endTime + timeSinceReply = story.timer.getElapsed() - r.getLastUtterance().endTime + delays = sorted(self.vars['delays'], key=lambda k: float(k['minReplyDuration']), reverse=True) + for delay in delays: + if replyDuration > float(delay['minReplyDuration']): + logger.log(LOG_BS, f"check delay duration is now {replyDuration}, already waiting for {timeSinceReply}, have to wait {delay['waitTime']}") + if timeSinceReply > float(delay['waitTime']): + return True + break # don't check other delays + # wait for delay to match + return False + + # There is a match and no delay say, person finished speaking. Go ahead sir! + return True def getLogSummary(self): return { @@ -245,6 +350,7 @@ class Story(object): self.commands = [] # queue of commands to send self.log = [] # all nodes/elements that are triggered self.currentMessage = None + self.currentReply = None self.timer = Stopwatch() self.isRunning = False @@ -312,6 +418,7 @@ class Story(object): self.events = [] # queue of received events self.commands = [] # queue of commands to send self.log = [] # all nodes/elements that are triggered + self.currentReply = None def add(self, obj): if obj.id in self.elements: @@ -369,27 +476,27 @@ class Story(object): if e['event'] == 'speech': # message is still playing: - if self.currentMessage and not self.lastMsgFinishTime: - #interrupt: -# FINISH THIS!!! -# self.hugvey.sendCommand({ -# 'action': 'stop', -# 'id': self.currentMessage.id, -# }) -# .... - pass + if self.currentMessage and not self.lastMsgFinishTime and self.previousReply and self.previousReply.forMessage.interruptCount < 4: + timeDiff = self.timer.getElapsed() - self.previousReply.forMessage.getFinishedTime() + if self.previousReply.forMessage.afterrunTime > timeDiff: + #interrupt only in given interval: + logger.warn("Interrupt message, replay {}".format(self.previousReply.forMessage.id)) + self.currentReply = self.previousReply + self.previousReply.forMessage.interruptCount += 1 + self.currentMessage = self.setCurrentMessage(self.previousReply.forMessage) # log if somebody starts speaking - # TODO: use pausing timer # TODO: implement interrupt - if self.lastSpeechStartTime is None or self.lastSpeechStartTime < self.lastMsgTime: - self.lastSpeechStartTime = e['time'] - + if self.currentReply is None: + self.currentReply= Reply(self.currentMessage) + + utterance = self.currentReply.getActiveUtterance(self.timer.getElapsed()) + utterance.setText(e['transcript']) + if e['is_final']: - # final result - self.lastSpeechEndTime = e['time'] - self.currentMessage.setReply(e['transcript'], self.timer.getElapsed()) - + utterance.setFinished(self.timer.getElapsed()) + + def _processDirections(self, directions): for direction in directions: for condition in direction.conditions: @@ -399,6 +506,7 @@ class Story(object): direction.setMetCondition(condition) self.addToLog(condition) self.addToLog(direction) + self.currentMessage.setFinished(self.timer.getElapsed()) self.setCurrentMessage(direction.msgTo) return direction @@ -439,9 +547,19 @@ class Story(object): logger.info("Stop renderer") def setCurrentMessage(self, message): + if self.currentMessage and not self.lastMsgFinishTime: + logger.info("Interrupt playback {}".format(self.currentMessage.id)) + # message is playing + self.hugvey.sendCommand({ + 'action': 'stop', + 'id': self.currentMessage.id, + }) + self.currentMessage = message self.lastMsgTime = time.time() self.lastMsgFinishTime = None # to be filled in by the event + self.previousReply = self.currentReply # we can use this for interrptions + self.currentReply = self.currentMessage.reply logger.info("Current message: ({0}) \"{1}\"".format( message.id, message.text)) diff --git a/hugvey_server.py b/hugvey_server.py index 61a3d8b..c866679 100644 --- a/hugvey_server.py +++ b/hugvey_server.py @@ -19,18 +19,20 @@ if __name__ == '__main__': argParser.add_argument( '--verbose', '-v', - action="store_true", + action='count', default=0 ) args = argParser.parse_args() # print(coloredlogs.DEFAULT_LOG_FORMAT) # exit() + loglevel = logging.NOTSET if args.verbose > 1 else logging.DEBUG if args.verbose > 0 else logging.INFO + coloredlogs.install( - level=logging.DEBUG if args.verbose else logging.INFO, + level=loglevel, # default: "%(asctime)s %(hostname)s %(name)s[%(process)d] %(levelname)s %(message)s" fmt="%(asctime)s %(hostname)s %(name)s[%(process)d,%(threadName)s] %(levelname)s %(message)s" ) - command = CentralCommand(debug_mode=args.verbose) + command = CentralCommand(debug_mode=args.verbose > 0) command.loadConfig(args.config) command.start() diff --git a/local b/local index e799809..16738c5 160000 --- a/local +++ b/local @@ -1 +1 @@ -Subproject commit e799809a59522d0f68f99f668a9ddf0f5f629912 +Subproject commit 16738c586f1938814b12ad50be2af06ffa2bf37e diff --git a/www/css/styles.css b/www/css/styles.css index d1e041f..bc0d516 100644 --- a/www/css/styles.css +++ b/www/css/styles.css @@ -16,6 +16,10 @@ body { .btn:hover, input[type="submit"]:hover { background: #666; } +input[type="number"] { + width: 80px; + text-align: right; } + @keyframes dash-animation { to { stroke-dashoffset: -1000; } } @@ -147,8 +151,8 @@ img.icon { display: block; margin: 0 -10px; padding: 5px 10px; } - #story label input, #story label select, #story label .label-value { - float: right; } + #story label input, #story label select, #story label .label-value, #story label .label-unit { + float: right; } #story label:nth-child(odd) { background-color: rgba(255, 255, 255, 0.3); } #story #msg { diff --git a/www/js/hugvey_console.js b/www/js/hugvey_console.js index 3f07794..b5c5f81 100644 --- a/www/js/hugvey_console.js +++ b/www/js/hugvey_console.js @@ -282,7 +282,7 @@ class Graph { let startAttributes = { 'name': msg['@id'] + '-start', - 'disabled': true, + 'readonly': 'readonly', 'type': 'checkbox', 'on': { 'change': this.getEditEventListener() @@ -354,6 +354,19 @@ class Graph { } } } ) + ), + crel( 'label', + crel( 'span', { + "title": "The time after the reply in which one can still interrupt to continue speaking" + }, 'Afterrun time' ), + crel( 'input', { + 'name': msg['@id'] + '-afterrunTime', + 'value': msg['afterrunTime'], + 'type': 'number', + 'on': { + 'change': this.getEditEventListener() + } + } ) ) ); msgEl.appendChild( msgInfoEl ); @@ -448,6 +461,7 @@ class Graph { for ( let conditionId of direction['conditions'] ) { let condition = this.getNodeById( conditionId ); + console.log(conditionId, condition); directionEl.appendChild( this.getEditConditionFormEl( condition, direction ) ); } @@ -464,11 +478,13 @@ class Graph { 'on': { 'click': ( e ) => { if(confirm("Do you want to remove this condition?")) { + console.log('remove condition for direction', condition, direction); panopticon.graph.rmCondition( condition, direction ); } } } - }, 'delete') + }, 'delete'), + ...this.getConditionInputsForType(condition['type'], condition['@id'], condition['vars']) ) let labelLabel = document.createElement( 'label' ); labelLabel.innerHTML = "Description"; @@ -481,51 +497,122 @@ class Graph { } ); labelLabel.appendChild( labelInput ); conditionEl.appendChild( labelLabel ); + - for ( let v in condition['vars'] ) { - let varLabel = document.createElement( 'label' ); - varLabel.innerHTML = v; - let varInput = document.createElement( 'input' ); - if ( v == 'seconds' ) { - varInput.type = 'number'; - } - varInput.name = `${condition['@id']}-vars.${v}`; - varInput.value = condition['vars'][v]; - varInput.addEventListener( 'change', this.getEditEventListener() ); - varLabel.appendChild( varInput ); - conditionEl.appendChild( varLabel ); - } +// for ( let v in condition['vars'] ) { +// let varLabel = document.createElement( 'label' ); +// varLabel.innerHTML = v; +// let varInput = document.createElement( 'input' ); +// if ( v == 'seconds' ) { +// varInput.type = 'number'; +// } +// varInput.name = `${condition['@id']}-vars.${v}`; +// varInput.value = condition['vars'][v]; +// varInput.addEventListener( 'change', this.getEditEventListener() ); +// varLabel.appendChild( varInput ); +// conditionEl.appendChild( varLabel ); +// } return conditionEl; } getConditionTypes() { - if ( typeof this.conditionTypes === 'undefined' ) { - // type: vars: attribtes for crel() - this.conditionTypes = { + return { 'timeout': { - 'seconds': { 'type': 'number', 'value': 10, 'min': 0, 'step': 0.1 } + 'seconds': { 'type': 'number', 'value': 10, 'min': 0, 'step': 0.1, 'unit': "s" } }, 'replyContains': { - 'regex': { 'value': '.+' } + 'delays.0.minReplyDuration': { 'type': 'number', 'value': 0, 'min': 0, 'step': 0.1, 'label': 'Delay 1 - reply duration', 'unit': "s", 'readonly': 'readonly' }, + 'delays.0.waitTime': { 'type': 'number', 'value': 3, 'min': 0, 'step': 0.1 , 'label': 'Delay 1 - wait time', 'unit': "s" }, + 'delays.1.minReplyDuration': { 'type': 'number', 'value': 5, 'min': 0, 'step': 0.1, 'label': 'Delay 2 - reply duration', 'unit': "s" }, + 'delays.1.waitTime': { 'type': 'number', 'value': 1, 'min': 0, 'step': 0.1, 'label': 'Delay 2 - time', 'unit': "s" }, + 'delays.2.minReplyDuration': { 'type': 'number', 'value': 10, 'min': 0, 'step': 0.1, 'label': 'Delay 3 - reply duration', 'unit': "s" }, + 'delays.2.waitTime': { 'type': 'number', 'value': 0, 'min': 0, 'step': 0.1, 'label': 'Delay 3 - time', 'unit': "s" }, + 'regex': { 'value': '','placeholder': "match any input" }, + 'instantMatch': { 'value': '', 'title': "When matched, don't wait for reply to finish. Instantly take this direction.", 'type':'checkbox' }, } - } - } - return this.conditionTypes; + }; } - - fillConditionFormForType( conditionForm, type ) { - conditionForm.innerHTML = ""; + + getConditionInputsForType( type, conditionId, values ) { + let inputs = []; let vars = this.getConditionTypes()[type]; for ( let v in vars ) { let attr = vars[v]; - attr['name'] = v; - conditionForm.appendChild( + attr['name'] = typeof conditionId == 'undefined' ? v : `${conditionId}-vars.${v}`; + if(typeof values != 'undefined') { + let value = this._getValueForPath(v, values); + attr['value'] = typeof value == 'undefined' ? "": value; + attr['on'] = { + 'change': this.getEditEventListener() + } ; + } else { + console.log(attr); + } + + inputs.push( crel( 'label', - crel( 'span', v ), + crel( 'span', { + 'title': attr.hasOwnProperty('title') ? attr['title'] : "" + }, attr.hasOwnProperty('label') ? attr['label'] : v ), crel( 'input', attr ) +// crel('span', {'class': 'label-unit'}, attr.hasOwnProperty('unit') ? attr['unit'] : "" ) ) ); } + return inputs; + } + + fillConditionFormForType( conditionForm, type, values ) { + conditionForm.innerHTML = ""; + let inputs = this.getConditionInputsForType(type); + for(let i of inputs) { + conditionForm.appendChild(i); + } + } + + _getValueForPath(path, vars) { + path = path.split( '.' ); // use vars.test to set ['vars']['test'] = value + let v = vars; + for ( let i = 0; i < path.length; i++ ) { + if(!isNaN(parseInt(path[i])) && isFinite(path[i])) { + // is int, use array, instead of obj + path[i] = parseInt(path[i]); + } + v = v[path[i]]; + if(typeof v == 'undefined') { + break; + } + } + return v; + } + + /** + * Save an array path (string) with a value to an object. Used to turn + * strings into nested arrays + * @param string path + * @param {any} value + * @param array|object vars + */ + _formPathToVars(path, value, vars) { + path = path.split( '.' ); // use vars.test to set ['vars']['test'] = value + console.log(path); + let res = vars; + for ( let i = 0; i < path.length; i++ ) { + if ( i == ( path.length - 1 ) ) { + console.log( 'last', path[i] ); + res[path[i]] = value; + } else { + if(!isNaN(parseInt(path[i+1])) && isFinite(path[i+1])) { + // is int, use array, instead of obj + path[i+1] = parseInt(path[i+1]); + } + if(typeof res[path[i]] == 'undefined') { + res[path[i]] = typeof path[i+1] == 'number' ? [] : {} + } + res = res[path[i]]; + } + } + return vars; } getAddConditionFormEl( direction ) { @@ -551,8 +638,10 @@ class Graph { form.delete( 'label' ); let vars = {}; for ( var pair of form.entries() ) { - vars[pair[0]] = pair[1]; + vars = g._formPathToVars(pair[0], pair[1], vars); } + // TODO: checkboxes + console.log("Createded", vars); g.addConditionForDirection( type, label, vars, direction ); } } @@ -602,12 +691,14 @@ class Graph { // TODO if ( typeof direction != 'undefined' ) { let pos = direction['conditions'].indexOf(id); + console.log('delete', id, 'on direction'); if(pos > -1) { direction['conditions'].splice(pos, 1); } for(let dir of this.directions) { - if(dir['conditions'].indexOf(id) > 0) { +// console.log('check if condition exists for dir', dir) + if(dir['conditions'].indexOf(id) > -1) { console.log("Condition still in use"); this.updateFromData(); this.build(); @@ -615,7 +706,8 @@ class Graph { return; } } - this._rmNode( id ); + console.log('No use, remove', condition) + this._rmNode( condition ); } else { for(let dir of this.directions) { let pos = dir['conditions'].indexOf(id); @@ -623,7 +715,9 @@ class Graph { dir['conditions'].splice(pos, 1); } } - this._rmNode( id ); + + console.log('remove condition?', id) + this._rmNode( condition ); } this.updateMsg(); } @@ -648,7 +742,8 @@ class Graph { "@id": this.language_code.substring( 0, 2 ) + "-n" + Date.now().toString( 36 ), "@type": "Msg", "text": "New", - "start": false + "start": false, + "afterrunTime": 0.5, } this.data.push( msg ); this.updateFromData(); diff --git a/www/scss/styles.scss b/www/scss/styles.scss index 1443cac..c1fc44b 100644 --- a/www/scss/styles.scss +++ b/www/scss/styles.scss @@ -23,6 +23,11 @@ body{ } } +input[type="number"] { + width: 80px; + text-align:right; +} + @keyframes dash-animation { to { stroke-dashoffset: -1000; @@ -244,9 +249,9 @@ img.icon{ display: block; margin: 0 -10px; padding: 5px 10px; - } - label input,label select, label .label-value{ - float: right; + input,select, .label-value, .label-unit{ + float: right; + } } label:nth-child(odd){ background-color: rgba(255,255,255,0.3);