r/hugvey
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

merged

Browse source
This commit is contained in:
Hugvey Central Command 2019-05-10 11:16:33 +02:00
commit da7a22b2c6
7 changed files with 200 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
README.md
View file

@ -195,4 +195,13 @@ Types of conditions:
TODO
times occured/only on n-th instance: determines the order of diversions of the same type (for Timeout and no_response). Starting at 1, as a diversion with value of 0 can occur always
times occured/only on n-th instance: determines the order of diversions of the same type (for Timeout and no_response). Starting at 1, as a diversion with value of 0 can occur always
# Network
## 4G Modem
Visit 192.168.5.1
The password is at the bottom of the device.

2
hugvey/central_command.py
View file

@ -96,6 +96,7 @@ class CentralCommand(object):
def loadLanguages(self):
logger.debug('load language files')
self.languages = {}
self.languageCache = {}
for lang in self.config['languages']:
lang_filename = os.path.join(self.config['web']['files_dir'], lang['file'])
@ -512,6 +513,7 @@ class HugveyState(object):
except asyncio.futures.TimeoutError as e:
# detect missing heartbeat:
if self.isConfigured and time.time() - self.isConfigured > 15:
self.logger.error("Hugvey did not send heartbeat.")
self.gone()
continue

4
hugvey/client.py
View file

@ -284,6 +284,10 @@ class CommandHandler(object):
# use duration for timing the popen duration (and redo it if needed)
duration = cmd['duration'] if 'duration' in cmd else None
self.playingMsgId = msgId
if self.playPopen:
logger.info("Interrupting playback of {}".format(self.playingMsgId))
self.playPopen.terminate()
err = None
if file is None and text is None:

9
hugvey/panopticon.py
View file

@ -152,7 +152,7 @@ def getUploadHandler(central_command):
self.finish()
def post(self):
print('upload')
logger.info('upload')
langCode = self.get_argument("language")
langFile = os.path.join(central_command.config['web']['files_dir'] , central_command.languageFiles[langCode])
@ -171,7 +171,7 @@ def getUploadHandler(central_command):
for i, data in enumerate(storyData):
if data['@id'] != msgId:
continue
if 'audio' in storyData[i] and os.path.exists(storyData[i]['audio']['file']):
if 'audio' in storyData[i] and storyData[i]['audio'] is not None and os.path.exists(storyData[i]['audio']['file']):
logger.info(f"Remove previous file {storyData[i]['audio']['file']} ({storyData[i]['audio']['original_name']})")
os.unlink(storyData[i]['audio']['file'])
@ -184,7 +184,8 @@ def getUploadHandler(central_command):
fp.write(audioFile['body'])
break
print(os.path.abspath(langFile))
# logger.info(os.path.abspath(langFile))
langFile = os.path.abspath(langFile)
with open(langFile, 'w') as json_fp:
logger.info(f'Save story to {langFile} {json_fp}')
json.dump(storyData, json_fp, indent=2)
@ -270,6 +271,6 @@ class Panopticon(object):
msg['args'] = items[2]
j = json.dumps(msg)
print(j)
logger.debug(j)
self.loop.add_callback(wsHandler.write_to_clients, j)

20
hugvey/speech/streamer.py
View file

@ -30,25 +30,25 @@ class AudioStreamer(object):
address = "tcp://{}:{}".format(self.address, self.port)
self.ctx = Context.instance()
self.socket = self.ctx.socket(zmq.SUB)
self.socket.setsockopt(zmq.RCVTIMEO, 6000) # timeout: 8 sec
self.socket.setsockopt(zmq.RCVTIMEO, 8000) # timeout: 8 sec
self.socket.subscribe('')
# self.socket.setsockopt(zmq.CONFLATE, 1)
self.socket.connect(address)
# s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.logger.info("Attempt connection on {}:{}".format(self.address, self.port))
self.logger.info("0mq SUBSCRIBE for audio stream on {}:{}".format(self.address, self.port))
# s.connect((self.address, self.port))
#
try:
while self.isRunning:
while self.isRunning:
try:
data = await self.socket.recv()
# self.logger.debug('chunk received')
# self.logger.debug('chunk received')
self.process(data)
except zmq.error.Again as timeout_e:
self.logger.warn("Timeout of audiostream. Hugvey shutdown?")
finally:
self.logger.info("Close socket on {}:{}".format(self.address, self.port))
self.socket.close()
except zmq.error.Again as timeout_e:
self.logger.warn("Timeout of audiostream. Hugvey shutdown?")
self.logger.info("Close socket on {}:{}".format(self.address, self.port))
self.socket.close()
def stop(self):
self.isRunning = False

155
hugvey/story.py
View file

@ -14,6 +14,7 @@ from zmq.asyncio import Context
import zmq
import wave
from pythonosc import udp_client
import random
mainLogger = logging.getLogger("hugvey")
logger = mainLogger.getChild("narrative")
@ -242,6 +243,12 @@ class Reply(object):
return True
return False
def getTimeSinceLastUtterance(self):
if not self.hasUtterances():
return None
return self.forMessage.story.timer.getElapsed() - self.getLastUtterance().lastUpdateTime
class Condition(object):
"""
A condition, basic conditions are built in, custom condition can be given by
@ -254,11 +261,14 @@ class Condition(object):
self.type = None
self.vars = {}
self.logInfo = None
self.originalJsonString = None
self.usedContainsDuration = None
@classmethod
def initFromJson(conditionClass, data, story):
condition = conditionClass(data['@id'])
condition.type = data['type']
condition.originalJsonString = json.dumps(data)
# TODO: should Condition be subclassed?
if data['type'] == "replyContains":
@ -404,7 +414,7 @@ class Condition(object):
delays = sorted(self.vars['delays'], key=lambda k: float(k['minReplyDuration']), reverse=True)
for delay in delays:
if replyDuration > float(delay['minReplyDuration']):
timeSinceReply = story.timer.getElapsed() - r.getLastUtterance().lastUpdateTime
timeSinceReply = r.getTimeSinceLastUtterance()
story.logger.log(LOG_BS, f"check delay duration is now {replyDuration}, already waiting for {timeSinceReply}, have to wait {delay['waitTime']}")
if timeSinceReply > float(delay['waitTime']):
# if variables are captured, only set them the moment the condition matches
@ -416,6 +426,7 @@ class Condition(object):
capturedVariables,
timeSinceReply
)
self.usedContainsDuration = float(delay['waitTime'])
return True
break # don't check other delays
# wait for delay to match
@ -487,6 +498,7 @@ class Diversion(object):
self.hasHit = False
self.disabled = False
self.type = type
self.counter = 0
if type == 'no_response':
self.method = self._divergeIfNoResponse
self.finaliseMethod = self._returnAfterNoResponse
@ -505,6 +517,8 @@ class Diversion(object):
if type == 'repeat':
self.method = self._divergeIfRepeatRequest
self.regex = re.compile(self.params['regex'])
if type == 'interrupt':
self.method = self._divergeIfInterrupted
if not self.method:
raise Exception("No valid type given for diversion")
@ -521,7 +535,7 @@ class Diversion(object):
'id': self.id,
}
async def divergeIfNeeded(self, story, direction: None):
async def divergeIfNeeded(self, story, direction = None):
"""
Validate if condition is met for the current story state
Returns True when diverging
@ -542,21 +556,30 @@ class Diversion(object):
direction.msgTo if direction else None,
direction if direction else None)
if r:
if self.type != 'repeat':
if self.type != 'repeat' and self.type !='interrupt':
# repeat diversion should be usable infinte times
self.hasHit = True
story.addToLog(self)
return r
def createReturnDirectionsTo(self, story, startMsg, returnMsg, originalDirection = None, inheritTiming = True):
def createReturnDirectionsTo(self, story, startMsg, returnMsg, originalDirection = None, inheritTiming = True, timeoutDuration = .5, replyContainsDurations = None):
"""
The finishes of this diversion's strand should point to the return message
with the right timeout/timing. If hit, this direction should also notify
this diversion.
replyContainsDurations: list formatted as in JSON
[{
"minReplyDuration": "0",
"waitTime": "3"
}]
"""
self.counter +=1
finishMessageIds = story.getFinishesForMsg(startMsg)
finalTimeoutDuration = 0.5
finalTimeoutDuration = timeoutDuration
finalContainsDurations = replyContainsDurations
#: :type story: Story
#: :type originalDirection: Direction
# story.directionsPerMsg[story.currentMessage.id]
@ -568,7 +591,8 @@ class Diversion(object):
for condition in originalDirection.conditions:
if condition.type == 'timeout':
finalTimeoutDuration = float(condition.vars['seconds'])
break
if condition.type == 'replyContains':
finalContainsDurations = json.loads(condition.originalJsonString)['vars']['delays']
i = 0
for msgId in finishMessageIds:
@ -578,21 +602,41 @@ class Diversion(object):
if not msg:
continue
direction = Direction(f"{self.id}-{i}", msg, returnMsg)
direction = Direction(f"{self.id}-{i}-{self.counter}", msg, returnMsg)
data = json.loads(f"""
{{
"@id": "{self.id}-c{i}",
"@id": "{self.id}-ct{i}-{self.counter}",
"@type": "Condition",
"type": "timeout",
"label": "",
"label": "Autogenerated Timeout",
"vars": {{
"seconds": "{finalTimeoutDuration}",
"onlyIfNoReply": false
"seconds": "{finalTimeoutDuration}"
}}
}}
""")
data['vars']['onlyIfNoReply'] = finalContainsDurations is not None
# TODO: also at replycontains if it exists, with the same timings
condition = Condition.initFromJson(data, story)
direction.addCondition(condition)
if finalContainsDurations is not None:
data2 = json.loads(f"""
{{
"@id": "{self.id}-cr{i}-{self.counter}",
"@type": "Condition",
"type": "replyContains",
"label": "Autogenerated Reply Contains",
"vars": {{
"regex": "",
"instantMatch": false
}} }}
""")
data2['vars']['delays'] = finalContainsDurations
condition2 = Condition.initFromJson(data2, story)
direction.addCondition(condition2)
story.add(condition2)
direction.isDiversionReturn = True # will clear the currentDiversion on story
story.logger.info(f"Created direction: {direction.id} {condition.id} with timeout {finalTimeoutDuration}s")
story.add(condition)
@ -653,6 +697,8 @@ class Diversion(object):
Participant doesn't speak for x consecutive replies (has had timeout)
"""
':type story: Story'
# TODO: disable check on msgFrom/msgTo to allow for own timing (2 sec)
# use story.currentReply.getTimeSinceLastUtterance() > 2
if story.currentDiversion or not msgFrom or not msgTo:
# don't do nested diversions
# if we remove this, don't forget to double check 'returnMessage'
@ -682,6 +728,7 @@ class Diversion(object):
story.logger.critical(f"Not a valid message id for diversion: {self.params['msgId']}")
return
# TODO: pick the direction with timeout as next Message.
self.returnMessage = msgTo
if self.params['returnAfterStrand']:
@ -700,21 +747,22 @@ class Diversion(object):
"""
Participant asks if message can be repeated.
"""
if not msgFrom or not msgTo:
return
# if not msgFrom or not msgTo:
# return
# TODO: how to handle this now we sometimes use different timings.
# Perhaps set isFinished when matching condition.
if story.currentReply is None or story.currentReply.isSpeaking():
if story.currentReply is None or story.currentReply.getTimeSinceLastUtterance() > 1:
return
r = self.regex.search(story.currentReply.getText())
print('repeat?', r)
if r is None:
return
logger.info(f"Diverge: request repeat {self.id}")
story.stats['diversions']['repeat'] += 1
await story.setCurrentMessage(msgFrom)
await story.setCurrentMessage(story.currentMessage)
return True
async def _divergeIfTimeout(self, story, msgFrom, msgTo, direction):
@ -722,6 +770,9 @@ class Diversion(object):
(1) last spoken at all
(2) or duration for this last reply only
"""
if story.currentDiversion:
return
if msgFrom or msgTo:
# not applicable a direction has been chosen
return
@ -790,7 +841,7 @@ class Diversion(object):
# no direction is here, as this diversion triggers before a direction is taken
self.createReturnDirectionsTo(story, msg, self.returnMessage, inheritTiming=False)
await story.setCurrentMessage(msg)
await story.setCurrentMessage(msg, allowReplyInterrupt=True)
story.currentDiversion = self
story.timer.setMark('last_diversion_timeout')
return True
@ -798,8 +849,31 @@ class Diversion(object):
async def _returnAfterTimeout(self, story):
story.logger.info(f"Finalise diversion: {self.id}")
# if self.params['returnAfterStrand']:
# await story.setCurrentMessage(self.returnMessage)
async def _divergeIfInterrupted(self, story, msgFrom, msgTo, direction):
"""
This is here as a placeholder for the interruption diversion.
These will however be triggered differently
"""
if story.currentDiversion or story.allowReplyInterrupt:
return False
msg = story.get(self.params['msgId'])
if msg is None:
story.logger.critical(f"Not a valid message id for diversion: {self.params['msgId']}")
return
self.returnMessage = story.currentMessage
# no direction is here, as this diversion triggers before a direction is taken
self.createReturnDirectionsTo(story, msg, self.returnMessage, inheritTiming=False, timeoutDuration=3, replyContainsDurations = [{
"minReplyDuration": "0",
"waitTime": "2"
}])
await story.setCurrentMessage(msg)
story.currentDiversion = self
return True
storyClasses = {
@ -875,9 +949,11 @@ class Story(object):
self.currentMessage = None
self.currentDiversion = None
self.currentReply = None
self.allowReplyInterrupt = False
self.timer = Stopwatch()
self.isRunning = False
self.diversions = []
self.interruptionDiversions = []
self.variables = {}
def pause(self):
@ -939,6 +1015,7 @@ class Story(object):
self.elements = {}
self.strands = {}
self.diversions = []
self.interruptionDiversions = []
self.directionsPerMsg = {}
self.startMessage = None # The entrypoint to the graph
self.variables = {}
@ -953,6 +1030,7 @@ class Story(object):
# self.logger.debug(self.directionsPerMsg)
self.diversions = [el for el in self.elements.values() if type(el) == Diversion]
self.interruptionDiversions = [el for el in self.elements.values() if type(el) == Diversion and el.type == 'interrupt']
if currentId:
self.currentMessage = self.get(currentId)
@ -1107,21 +1185,18 @@ class Story(object):
# messages that come in, in the case google is faster than our playbackFinish event.
# (if this setup doesn't work, try to test on self.lastMsgFinish time anyway)
# it keeps tricky with all these run conditions
self.logger.info("ignore speech while playing message")
continue
# if len(self.interruptionDiversions) and not self.currentDiversion and not self.allowReplyInterrupt:
# self.logger.warn("diverge when speech during playing message")
# diversion = random.choice(self.interruptionDiversions)
# #: :type diversion: Diversion
# r = await diversion.divergeIfNeeded(self, None)
# print(r) # is always needed :-)
# else:
self.logger.info("ignore speech during playing message")
continue
# message is still playing:
if self.currentMessage and not self.lastMsgFinishTime and self.previousReply and self.previousReply.forMessage.interruptCount < 4:
timeDiff = self.timer.getElapsed() - self.previousReply.forMessage.getFinishedTime()
if self.previousReply.forMessage.afterrunTime > timeDiff:
#interrupt only in given interval:
self.logger.warn("Interrupt message, replay {}".format(self.previousReply.forMessage.id))
self.currentReply = self.previousReply
self.previousReply.forMessage.interruptCount += 1
self.currentMessage = await self.setCurrentMessage(self.previousReply.forMessage, self.previousReply)
# log if somebody starts speaking
# TODO: implement interrupt
if self.currentReply is None:
self.logger.info("Start speaking")
self.currentReply= Reply(self.currentMessage)
@ -1143,13 +1218,15 @@ class Story(object):
async def _processDirections(self, directions):
':type directions: list(Direction)'
chosenDirection = None
metCondition = None
for direction in directions:
for condition in direction.conditions:
if condition.isMet(self):
self.logger.info("Condition is met: {0}, going to {1}".format(
condition.id, direction.msgTo.id))
self.hugvey.eventLogger.info("condition: {0}".format(condition.id))
self.hugvey.eventLogger.info("direction: {0}".format(direction.id))
self.hugvey.eventLogger.info("direction: {0}".format(direction.id))
metCondition = condition
direction.setMetCondition(condition)
self.addToLog(condition)
self.addToLog(direction)
@ -1158,11 +1235,19 @@ class Story(object):
isDiverging = await self._processDiversions(chosenDirection)
allowReplyInterrupt = False
# in some cases, conditions should be allowed to interrupt the reply
if metCondition:
if metCondition.type == 'timeout' and not ('onlyIfNoReply' in metCondition.vars and metCondition.vars['onlyIfNoReply']):
allowReplyInterrupt = True
if metCondition.usedContainsDuration is not None and metCondition.usedContainsDuration < 0.1:
allowReplyInterrupt = True
if not isDiverging and chosenDirection:
if chosenDirection.isDiversionReturn and self.currentDiversion:
await self.currentDiversion.finalise(self)
await self.setCurrentMessage(chosenDirection.msgTo)
await self.setCurrentMessage(chosenDirection.msgTo, allowReplyInterrupt=allowReplyInterrupt)
return chosenDirection
@ -1180,7 +1265,8 @@ class Story(object):
activeNoResponseDiv = None
for diversion in self.diversions:
#: :type diversion: Diversion
if diversion.disabled or diversion.hasHit:
if diversion.disabled or diversion.hasHit or diversion.type == 'interrupt':
# interruptions are triggered somewhere else.
continue
if diversion.type == 'timeout':
@ -1269,7 +1355,7 @@ class Story(object):
self.logger.debug("Stop renderer")
async def setCurrentMessage(self, message, useReply = None):
async def setCurrentMessage(self, message, useReply = None, allowReplyInterrupt = False):
"""
Use Reply allows to pre-initiate a reply to use with the message. This is used eg. when doing an interruption.
"""
@ -1288,6 +1374,7 @@ class Story(object):
self.lastMsgTime = time.time()
self.lastMsgFinishTime = None # to be filled in by the event
self.lastMsgStartTime = None # to be filled in by the event
self.allowReplyInterrupt = allowReplyInterrupt
# if not reset:
self.previousReply = self.currentReply # we can use this for interrptions

51
www/js/hugvey_console.js
View file

@ -367,6 +367,9 @@ class Graph {
div['params']['msgId'] = "";
div['params']['notForColor'] = "";
}
else if(type == 'interrupt') {
div['params']['msgId'] = "";
}
else if(type == 'timeout') {
div['params']['interval'] = 20;
div['params']['timesOccured'] = 0;
@ -382,7 +385,7 @@ class Graph {
alert('invalid type for diversion');
}
if(type != 'repeat') {
if(type != 'repeat' && type != 'interrupt') {
div['params']['notAfterMsgId'] = "";
}
@ -403,7 +406,7 @@ class Graph {
let msgEl = document.getElementById( 'msg' );
msgEl.innerHTML = "";
let divsNoResponse =[], divsRepeat = [], divsReplyContains = [], divsTimeouts = [];
let divsNoResponse =[], divsRepeat = [], divsReplyContains = [], divsTimeouts = [], divsInterrupts = [];
for(let div of this.diversions) {
let notAfterMsgIdEl = "";
@ -691,9 +694,37 @@ class Graph {
)
));
}
if(div['type'] == 'interrupt'){
let msgOptions = [crel('option',"")];
let starts = this.messages.filter( m => m.hasOwnProperty('start') && m['start'] == true);
for(let startMsg of starts) {
let optionParams = {};
if(div['params']['msgId'] == startMsg['@id']) {
optionParams['selected'] = 'selected';
}
msgOptions.push(crel('option', optionParams , startMsg['@id']));
}
divsInterrupts.push(crel(
'div', {'class': 'diversion'},
crel('h3', div['@id']),
crel(
'div', {
'class':'btn btn--delete',
'on': {
'click': (e) => this.deleteDiversion(div)
}
}, 'Delete diversion'),
crel('label', 'Go to (start message)',
crel('select', {'on': {
'change': (e) => div['params']['msgId'] = e.target.value
}}, ...msgOptions)
)
));
}
}
console.log(divsReplyContains, divsNoResponse, divsRepeat, divsTimeouts);
console.log(divsReplyContains, divsNoResponse, divsRepeat, divsTimeouts, divsInterrupts);
let divEl = crel(
'div',
@ -753,6 +784,20 @@ class Graph {
'New case for timeout'
)
)
// ,
// crel('div',
// crel('h2', 'Interruptions (random pick)'),
// ...divsInterrupts,
// crel('div',
// {
// 'class': 'btn',
// 'on': {
// 'click': (e) => this.createDiversion('interrupt')
// }
// },
// 'New case for Interrupt'
// )
// )
);
msgEl.appendChild(divEl);