merged

README.md

@@ -196,3 +196,12 @@ TODO
 
 
 times occured/only on n-th instance: determines the order of diversions of the same type (for Timeout and no_response). Starting at 1, as a diversion with value of 0 can occur always
+
+
+
+# Network
+
+## 4G Modem
+
+Visit 192.168.5.1
+The password is at the bottom of the device.

hugvey/central_command.py

@@ -96,6 +96,7 @@ class CentralCommand(object):
     def loadLanguages(self):
         logger.debug('load language files')
         self.languages = {}
+        self.languageCache = {}
 
         for lang in self.config['languages']:
             lang_filename = os.path.join(self.config['web']['files_dir'], lang['file'])
@@ -512,6 +513,7 @@ class HugveyState(object):
             except asyncio.futures.TimeoutError as e:
                 # detect missing heartbeat:
                 if  self.isConfigured and time.time() - self.isConfigured > 15:
+                    self.logger.error("Hugvey did not send heartbeat.")
                     self.gone()
                 continue
             

hugvey/client.py

@@ -285,6 +285,10 @@ class CommandHandler(object):
         duration = cmd['duration'] if 'duration' in cmd else None
         self.playingMsgId = msgId
         
+        if self.playPopen:
+            logger.info("Interrupting playback of {}".format(self.playingMsgId))
+            self.playPopen.terminate()
+
         err = None
         if file is None and text is None:
             logger.critical("No file nor text given: {}".format(cmd))

hugvey/panopticon.py

@@ -152,7 +152,7 @@ def getUploadHandler(central_command):
             self.finish()
 
         def post(self):
-            print('upload')
+            logger.info('upload')
             langCode = self.get_argument("language")
             langFile = os.path.join(central_command.config['web']['files_dir'] , central_command.languageFiles[langCode])
             
@@ -171,7 +171,7 @@ def getUploadHandler(central_command):
                 for i, data in enumerate(storyData):
                     if data['@id'] != msgId:
                         continue
-                    if 'audio' in storyData[i] and os.path.exists(storyData[i]['audio']['file']):
+                    if 'audio' in storyData[i] and storyData[i]['audio'] is not None and os.path.exists(storyData[i]['audio']['file']):
                         logger.info(f"Remove previous file {storyData[i]['audio']['file']} ({storyData[i]['audio']['original_name']})")
                         os.unlink(storyData[i]['audio']['file'])
                         
@@ -184,7 +184,8 @@ def getUploadHandler(central_command):
                         fp.write(audioFile['body'])
                     break
             
-            print(os.path.abspath(langFile))
+#             logger.info(os.path.abspath(langFile))
+            langFile = os.path.abspath(langFile)
             with open(langFile, 'w') as json_fp:
                 logger.info(f'Save story to {langFile} {json_fp}')
                 json.dump(storyData, json_fp, indent=2)
@@ -270,6 +271,6 @@ class Panopticon(object):
                 msg['args'] = items[2]
 
             j = json.dumps(msg)
-            print(j)
+            logger.debug(j)
             self.loop.add_callback(wsHandler.write_to_clients, j)
             

hugvey/speech/streamer.py

@@ -30,25 +30,25 @@ class AudioStreamer(object):
         address = "tcp://{}:{}".format(self.address, self.port)
         self.ctx = Context.instance()
         self.socket = self.ctx.socket(zmq.SUB)
-        self.socket.setsockopt(zmq.RCVTIMEO, 6000) # timeout: 8 sec
+        self.socket.setsockopt(zmq.RCVTIMEO, 8000) # timeout: 8 sec
         self.socket.subscribe('')
 #         self.socket.setsockopt(zmq.CONFLATE, 1)
         self.socket.connect(address)
         
 #         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.logger.info("Attempt connection on {}:{}".format(self.address, self.port))
+        self.logger.info("0mq SUBSCRIBE for audio stream on {}:{}".format(self.address, self.port))
 #         s.connect((self.address, self.port))
 #     
-        try:
+        while self.isRunning:
-            while self.isRunning:
+            try:
                 data = await self.socket.recv()
-    #             self.logger.debug('chunk received')
+#                 self.logger.debug('chunk received')
                 self.process(data)
-        except zmq.error.Again as timeout_e:
+            except zmq.error.Again as timeout_e:
-            self.logger.warn("Timeout of audiostream. Hugvey shutdown?")
+                self.logger.warn("Timeout of audiostream. Hugvey shutdown?")
-        finally:
+        
-            self.logger.info("Close socket on {}:{}".format(self.address, self.port))
+        self.logger.info("Close socket on {}:{}".format(self.address, self.port))
-            self.socket.close()
+        self.socket.close()
     
     def stop(self):
         self.isRunning = False

hugvey/story.py

@@ -14,6 +14,7 @@ from zmq.asyncio import Context
 import zmq
 import wave
 from pythonosc import udp_client
+import random
 
 mainLogger = logging.getLogger("hugvey")
 logger = mainLogger.getChild("narrative")
@@ -242,6 +243,12 @@ class Reply(object):
             return True
         return False
     
+    def getTimeSinceLastUtterance(self):
+        if not self.hasUtterances():
+            return None
+        
+        return self.forMessage.story.timer.getElapsed() - self.getLastUtterance().lastUpdateTime
+    
 class Condition(object):
     """
     A condition, basic conditions are built in, custom condition can be given by
@@ -254,11 +261,14 @@ class Condition(object):
         self.type = None
         self.vars = {}
         self.logInfo = None
+        self.originalJsonString = None
+        self.usedContainsDuration = None
 
     @classmethod
     def initFromJson(conditionClass, data, story):
         condition = conditionClass(data['@id'])
         condition.type = data['type']
+        condition.originalJsonString = json.dumps(data)
         
         # TODO: should Condition be subclassed?
         if data['type'] == "replyContains":
@@ -404,7 +414,7 @@ class Condition(object):
             delays = sorted(self.vars['delays'], key=lambda k: float(k['minReplyDuration']), reverse=True)
             for delay in delays:
                 if replyDuration > float(delay['minReplyDuration']):
-                    timeSinceReply = story.timer.getElapsed() - r.getLastUtterance().lastUpdateTime
+                    timeSinceReply = r.getTimeSinceLastUtterance()
                     story.logger.log(LOG_BS, f"check delay duration is now {replyDuration}, already waiting for {timeSinceReply}, have to wait {delay['waitTime']}")
                     if timeSinceReply > float(delay['waitTime']):
                         # if variables are captured, only set them the moment the condition matches
@@ -416,6 +426,7 @@ class Condition(object):
                             capturedVariables,
                             timeSinceReply 
                         )
+                        self.usedContainsDuration = float(delay['waitTime'])
                         return True
                     break # don't check other delays
             # wait for delay to match
@@ -487,6 +498,7 @@ class Diversion(object):
         self.hasHit = False
         self.disabled = False
         self.type = type
+        self.counter = 0
         if type == 'no_response':
             self.method = self._divergeIfNoResponse
             self.finaliseMethod = self._returnAfterNoResponse
@@ -505,6 +517,8 @@ class Diversion(object):
         if type == 'repeat':
             self.method = self._divergeIfRepeatRequest
             self.regex = re.compile(self.params['regex'])
+        if type == 'interrupt':
+            self.method = self._divergeIfInterrupted
             
         if not self.method:
             raise Exception("No valid type given for diversion")
@@ -521,7 +535,7 @@ class Diversion(object):
             'id': self.id,
         }
     
-    async def divergeIfNeeded(self, story, direction: None):
+    async def divergeIfNeeded(self, story, direction = None):
         """
         Validate if condition is met for the current story state
         Returns True when diverging
@@ -542,21 +556,30 @@ class Diversion(object):
                 direction.msgTo if direction else None,
                 direction if direction else None)
         if r:
-            if self.type != 'repeat':
+            if self.type != 'repeat' and self.type !='interrupt':
                 # repeat diversion should be usable infinte times
                 self.hasHit = True
                 
             story.addToLog(self)
         return r
     
-    def createReturnDirectionsTo(self, story, startMsg, returnMsg, originalDirection = None, inheritTiming = True):
+    def createReturnDirectionsTo(self, story, startMsg, returnMsg, originalDirection = None, inheritTiming = True, timeoutDuration = .5, replyContainsDurations = None):
         """
         The finishes of this diversion's strand should point to the return message
         with the right timeout/timing. If hit, this direction should also notify
         this diversion.
+        
+        replyContainsDurations: list formatted as in JSON
+            [{
+              "minReplyDuration": "0",
+              "waitTime": "3"
+            }]
         """
+        self.counter  +=1
+        
         finishMessageIds = story.getFinishesForMsg(startMsg)
-        finalTimeoutDuration = 0.5
+        finalTimeoutDuration = timeoutDuration
+        finalContainsDurations = replyContainsDurations
         #: :type story: Story
         #: :type originalDirection: Direction
 #         story.directionsPerMsg[story.currentMessage.id]
@@ -568,7 +591,8 @@ class Diversion(object):
                 for condition in originalDirection.conditions:
                     if condition.type == 'timeout':
                         finalTimeoutDuration = float(condition.vars['seconds'])
-                        break
+                    if condition.type == 'replyContains':
+                        finalContainsDurations = json.loads(condition.originalJsonString)['vars']['delays']
         
         i = 0
         for msgId in finishMessageIds:
@@ -578,21 +602,41 @@ class Diversion(object):
             if not msg:
                 continue
             
-            direction = Direction(f"{self.id}-{i}", msg, returnMsg)
+            direction = Direction(f"{self.id}-{i}-{self.counter}", msg, returnMsg)
             data = json.loads(f"""
             {{
-                "@id": "{self.id}-c{i}",
+                "@id": "{self.id}-ct{i}-{self.counter}",
                 "@type": "Condition",
                 "type": "timeout",
-                "label": "",
+                "label": "Autogenerated Timeout",
                 "vars": {{
-                  "seconds": "{finalTimeoutDuration}",
+                  "seconds": "{finalTimeoutDuration}"
-                  "onlyIfNoReply": false
                 }}
               }}
             """)
+            data['vars']['onlyIfNoReply'] = finalContainsDurations is not None
+            
+            # TODO: also at replycontains if it exists, with the same timings
             condition = Condition.initFromJson(data, story)
             direction.addCondition(condition)
+            
+            if finalContainsDurations is not None:
+                data2 = json.loads(f"""
+                {{
+                    "@id": "{self.id}-cr{i}-{self.counter}",
+                    "@type": "Condition",
+                    "type": "replyContains",
+                    "label": "Autogenerated Reply Contains",
+                    "vars": {{
+                      "regex": "",
+                      "instantMatch": false
+                  }} }}
+                """)
+                data2['vars']['delays'] = finalContainsDurations
+                condition2 = Condition.initFromJson(data2, story)
+                direction.addCondition(condition2)
+                story.add(condition2)
+            
             direction.isDiversionReturn = True # will clear the currentDiversion on story
             story.logger.info(f"Created direction: {direction.id} {condition.id} with timeout {finalTimeoutDuration}s")
             story.add(condition)
@@ -653,6 +697,8 @@ class Diversion(object):
         Participant doesn't speak for x consecutive replies (has had timeout)
         """
         ':type story: Story'
+        # TODO: disable check on msgFrom/msgTo to allow for own timing (2 sec)
+        # use story.currentReply.getTimeSinceLastUtterance() > 2
         if story.currentDiversion or not msgFrom or not msgTo:
             # don't do nested diversions
             # if we remove this, don't forget to double check 'returnMessage'
@@ -682,6 +728,7 @@ class Diversion(object):
             story.logger.critical(f"Not a valid message id for diversion: {self.params['msgId']}")
             return
         
+        # TODO: pick the direction with timeout as next Message.
         self.returnMessage = msgTo
         
         if self.params['returnAfterStrand']:
@@ -700,21 +747,22 @@ class Diversion(object):
         """
         Participant asks if message can be repeated.
         """
-        if not msgFrom or not msgTo:
+#         if not msgFrom or not msgTo:
-            return
+#             return
         
         # TODO: how to handle this now we sometimes use different timings.
         # Perhaps set isFinished when matching condition.
-        if story.currentReply is None or story.currentReply.isSpeaking():
+        if story.currentReply is None or story.currentReply.getTimeSinceLastUtterance() > 1:
             return
         
         r = self.regex.search(story.currentReply.getText())
+        print('repeat?', r)
         if r is None:
             return
         
         logger.info(f"Diverge: request repeat {self.id}")
         story.stats['diversions']['repeat'] += 1
-        await story.setCurrentMessage(msgFrom)
+        await story.setCurrentMessage(story.currentMessage)
         return True
     
     async def _divergeIfTimeout(self, story, msgFrom, msgTo, direction):
@@ -722,6 +770,9 @@ class Diversion(object):
         (1) last spoken at all
         (2) or duration for this last reply only
         """
+        if story.currentDiversion:
+            return
+        
         if msgFrom or msgTo:
             # not applicable a direction has been chosen
             return
@@ -790,7 +841,7 @@ class Diversion(object):
             # no direction is here, as this diversion triggers before a direction is taken
             self.createReturnDirectionsTo(story, msg, self.returnMessage, inheritTiming=False)
             
-        await story.setCurrentMessage(msg)
+        await story.setCurrentMessage(msg, allowReplyInterrupt=True)
         story.currentDiversion = self
         story.timer.setMark('last_diversion_timeout')
         return True
@@ -798,8 +849,31 @@ class Diversion(object):
     async def _returnAfterTimeout(self, story):
         story.logger.info(f"Finalise diversion: {self.id}")
         
-#         if self.params['returnAfterStrand']:
+
-#             await story.setCurrentMessage(self.returnMessage)
+    async def _divergeIfInterrupted(self, story, msgFrom, msgTo, direction):
+        """
+        This is here as a placeholder for the interruption diversion.
+        These will however be triggered differently
+        """
+        
+        if story.currentDiversion or story.allowReplyInterrupt:
+            return False
+        
+        msg = story.get(self.params['msgId'])
+        if msg is None:
+            story.logger.critical(f"Not a valid message id for diversion: {self.params['msgId']}")
+            return
+        
+        self.returnMessage = story.currentMessage
+        # no direction is here, as this diversion triggers before a direction is taken
+        self.createReturnDirectionsTo(story, msg, self.returnMessage, inheritTiming=False, timeoutDuration=3, replyContainsDurations = [{
+              "minReplyDuration": "0",
+              "waitTime": "2"
+            }])
+        await story.setCurrentMessage(msg)
+        story.currentDiversion = self
+        
+        return True
 
 
 storyClasses = {
@@ -875,9 +949,11 @@ class Story(object):
         self.currentMessage = None
         self.currentDiversion = None
         self.currentReply = None
+        self.allowReplyInterrupt = False
         self.timer = Stopwatch()
         self.isRunning = False
         self.diversions = []
+        self.interruptionDiversions = []
         self.variables = {}
     
     def pause(self):
@@ -939,6 +1015,7 @@ class Story(object):
         self.elements = {}
         self.strands = {}
         self.diversions = []
+        self.interruptionDiversions = []
         self.directionsPerMsg = {}
         self.startMessage = None  # The entrypoint to the graph
         self.variables = {}
@@ -953,6 +1030,7 @@ class Story(object):
 #         self.logger.debug(self.directionsPerMsg)
         
         self.diversions = [el for el in self.elements.values() if type(el) == Diversion]
+        self.interruptionDiversions = [el for el in self.elements.values() if type(el) == Diversion and el.type == 'interrupt']
 
         if currentId:
             self.currentMessage = self.get(currentId)
@@ -1107,21 +1185,18 @@ class Story(object):
                     # messages that come in, in the case google is faster than our playbackFinish event.
                     # (if this setup doesn't work, try to test on self.lastMsgFinish time anyway)
                     # it keeps tricky with all these run conditions
-                    self.logger.info("ignore speech while playing message")
+#                     if len(self.interruptionDiversions) and not self.currentDiversion and not self.allowReplyInterrupt:
-                    continue
+#                         self.logger.warn("diverge when speech during playing message")
+#                         diversion = random.choice(self.interruptionDiversions)
+#                         #: :type diversion: Diversion
+#                         r = await diversion.divergeIfNeeded(self, None)
+#                         print(r) # is always needed :-)
+#                     else:
+                        self.logger.info("ignore speech during playing message")
+                        continue
                 
-#                 message is still playing:
-                if self.currentMessage and not self.lastMsgFinishTime and self.previousReply and self.previousReply.forMessage.interruptCount < 4:
-                    timeDiff = self.timer.getElapsed() - self.previousReply.forMessage.getFinishedTime()
-                    if self.previousReply.forMessage.afterrunTime > timeDiff:
-                        #interrupt only in given interval:
-                        self.logger.warn("Interrupt message, replay {}".format(self.previousReply.forMessage.id))
-                        self.currentReply = self.previousReply
-                        self.previousReply.forMessage.interruptCount += 1
-                        self.currentMessage = await self.setCurrentMessage(self.previousReply.forMessage, self.previousReply)
                 
                 # log if somebody starts speaking
-                # TODO: implement interrupt
                 if self.currentReply is None:
                     self.logger.info("Start speaking")
                     self.currentReply= Reply(self.currentMessage)
@@ -1143,6 +1218,7 @@ class Story(object):
     async def _processDirections(self, directions):
         ':type directions: list(Direction)'
         chosenDirection = None
+        metCondition = None
         for direction in directions:
             for condition in direction.conditions:
                 if condition.isMet(self):
@@ -1150,6 +1226,7 @@ class Story(object):
                         condition.id, direction.msgTo.id))
                     self.hugvey.eventLogger.info("condition: {0}".format(condition.id))
                     self.hugvey.eventLogger.info("direction: {0}".format(direction.id))
+                    metCondition = condition
                     direction.setMetCondition(condition)
                     self.addToLog(condition)
                     self.addToLog(direction)
@@ -1158,11 +1235,19 @@ class Story(object):
                     
         isDiverging = await self._processDiversions(chosenDirection)
         
+        allowReplyInterrupt = False
+        # in some cases, conditions should be allowed to interrupt the reply
+        if metCondition:
+            if metCondition.type == 'timeout' and not ('onlyIfNoReply' in metCondition.vars and metCondition.vars['onlyIfNoReply']):
+                allowReplyInterrupt = True
+            if metCondition.usedContainsDuration is not None and metCondition.usedContainsDuration < 0.1:
+                allowReplyInterrupt = True
+        
         if not isDiverging and chosenDirection:
             if chosenDirection.isDiversionReturn and self.currentDiversion:
                 await self.currentDiversion.finalise(self)
                 
-            await self.setCurrentMessage(chosenDirection.msgTo)
+            await self.setCurrentMessage(chosenDirection.msgTo, allowReplyInterrupt=allowReplyInterrupt)
             
         return chosenDirection
     
@@ -1180,7 +1265,8 @@ class Story(object):
         activeNoResponseDiv = None
         for diversion in self.diversions:
             #: :type diversion: Diversion
-            if diversion.disabled or diversion.hasHit:
+            if diversion.disabled or diversion.hasHit or diversion.type == 'interrupt':
+                # interruptions are triggered somewhere else.
                 continue
             
             if diversion.type == 'timeout':
@@ -1269,7 +1355,7 @@ class Story(object):
 
         self.logger.debug("Stop renderer")
 
-    async def setCurrentMessage(self, message, useReply = None):
+    async def setCurrentMessage(self, message, useReply = None, allowReplyInterrupt = False):
         """
         Use Reply allows to pre-initiate a reply to use with the message. This is used eg. when doing an interruption.
         """
@@ -1288,6 +1374,7 @@ class Story(object):
         self.lastMsgTime = time.time()
         self.lastMsgFinishTime = None  # to be filled in by the event
         self.lastMsgStartTime = None  # to be filled in by the event
+        self.allowReplyInterrupt = allowReplyInterrupt
         
 #         if not reset:
         self.previousReply = self.currentReply # we can use this for interrptions

www/js/hugvey_console.js

@@ -367,6 +367,9 @@ class Graph {
     		div['params']['msgId'] = "";
     		div['params']['notForColor'] = "";
     	}
+    	else if(type == 'interrupt') {
+    		div['params']['msgId'] = "";
+    	}
     	else if(type == 'timeout') {
     		div['params']['interval'] = 20;
     		div['params']['timesOccured'] = 0;
@@ -382,7 +385,7 @@ class Graph {
     		alert('invalid type for diversion');
     	}
     	
-    	if(type != 'repeat') {
+    	if(type != 'repeat' && type != 'interrupt') {
     		div['params']['notAfterMsgId'] = "";
     	}
     	
@@ -403,7 +406,7 @@ class Graph {
     	let msgEl = document.getElementById( 'msg' );
         msgEl.innerHTML = "";
         
-        let divsNoResponse =[], divsRepeat = [], divsReplyContains = [], divsTimeouts = [];
+        let divsNoResponse =[], divsRepeat = [], divsReplyContains = [], divsTimeouts = [], divsInterrupts = [];
         for(let div of this.diversions) {
 
         	let notAfterMsgIdEl = "";
@@ -691,9 +694,37 @@ class Graph {
         			)
         		));
         	}
+        	if(div['type'] == 'interrupt'){
+        		let msgOptions = [crel('option',"")];
+        		let starts = this.messages.filter( m => m.hasOwnProperty('start') && m['start'] == true);
+        		for(let startMsg of starts) {
+        			let optionParams = {};
+        			if(div['params']['msgId'] == startMsg['@id']) {
+        				optionParams['selected'] = 'selected';
+        			}
+        			msgOptions.push(crel('option', optionParams , startMsg['@id']));
+        		}
+        		
+        		divsInterrupts.push(crel(
+        			'div', {'class': 'diversion'},
+        			crel('h3', div['@id']),
+        			crel(
+        				'div', {
+        					'class':'btn btn--delete',
+        					'on': {
+        						'click': (e) => this.deleteDiversion(div)
+        					}
+        				}, 'Delete diversion'),
+        				crel('label', 'Go to (start message)',
+                				crel('select', {'on': {
+            						'change': (e) => div['params']['msgId'] = e.target.value
+            					}}, ...msgOptions)
+        			)
+        		));
+        	}
         }
         
-        console.log(divsReplyContains, divsNoResponse, divsRepeat, divsTimeouts);
+        console.log(divsReplyContains, divsNoResponse, divsRepeat, divsTimeouts, divsInterrupts);
         
         let divEl = crel(
         	'div',
@@ -753,6 +784,20 @@ class Graph {
 	    			'New case for timeout'
         		)
         	)
+//        	,
+//        	crel('div',
+//        		crel('h2', 'Interruptions (random pick)'),
+//        		...divsInterrupts,
+//        		crel('div',
+//        			{
+//	        			'class': 'btn',
+//	        			'on': {
+//	        				'click': (e) => this.createDiversion('interrupt')
+//	        			}
+//	        		},	
+//	    			'New case for Interrupt'
+//        		)
+//        	)
         );
         
         msgEl.appendChild(divEl);