r/hugvey
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Colors and SFX

Browse source
This commit is contained in:
Ruben van de Ven 2019-02-28 18:58:03 +01:00
parent a798b3b638
commit 1563f9d1af
5 changed files with 194 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

129
hugvey/client.py
View file

@ -10,11 +10,12 @@ import yaml
import zmq import zmq
from zmq.asyncio import Context from zmq.asyncio import Context
import sys import sys
from hugvey.communication import LOG_BS
try: try:
import alsaaudio import alsaaudio
except ImportError: except ImportError:
print("No volume settings available") print("No volume settings available")
from .communication import zmqReceive, zmqSend, getTopic from .communication import zmqReceive, zmqSend, getTopic
import subprocess import subprocess
@ -22,8 +23,10 @@ import subprocess
logger = logging.getLogger("client") logger = logging.getLogger("client")
class VoiceServer(object): class VoiceServer(object):
"""A UDP server, providing mic data at 16 kHz""" """A UDP server, providing mic data at 16 kHz"""
def __init__(self, loop, hugvey, voice_port: int, input_rate: int, input_name: str = None, target_rate: int = 16000): def __init__(self, loop, hugvey, voice_port: int, input_rate: int, input_name: str = None, target_rate: int = 16000):
self.voice_port = voice_port self.voice_port = voice_port
self.input_rate = input_rate self.input_rate = input_rate
@ -45,28 +48,31 @@ class VoiceServer(object):
dev = self.p.get_device_info_by_index(i) dev = self.p.get_device_info_by_index(i)
if input_device_idx is None and dev['maxInputChannels'] > 0: if input_device_idx is None and dev['maxInputChannels'] > 0:
if (self.input_name and self.input_name in dev['name']) or \ if (self.input_name and self.input_name in dev['name']) or \
(not self.input_name and dev['name'] != 'default'): (not self.input_name and dev['name'] != 'default'):
input_device_idx = dev['index'] input_device_idx = dev['index']
logger.info("Use device {0}: {1}".format(dev['index'],dev['name'])) logger.info("Use device {0}: {1}".format(
logger.debug("{} {:0d} {}".format("* " if input_device_idx == i else "- ", i, dev['name'])) dev['index'], dev['name']))
logger.debug("{} {:0d} {}".format(
"* " if input_device_idx == i else "- ", i, dev['name']))
return input_device_idx return input_device_idx
def onBuffer(self, in_data, frame_count, time_info, status): def onBuffer(self, in_data, frame_count, time_info, status):
if self.input_rate == self.target_rate: if self.input_rate == self.target_rate:
f = in_data f = in_data
else: else:
# chunk 4096, with 2 bytes per frame gives len(in_data) of 8192 # chunk 4096, with 2 bytes per frame gives len(in_data) of 8192
# rate converted 44k1 -> 16k gives len(f) == 2972 (16/44.1 * 8192) # rate converted 44k1 -> 16k gives len(f) == 2972 (16/44.1 * 8192)
f, self.laststate = audioop.ratecv(in_data, 2, 1, self.input_rate, self.target_rate, self.laststate) f, self.laststate = audioop.ratecv(
in_data, 2, 1, self.input_rate, self.target_rate, self.laststate)
try: try:
if self.hugvey.cmd_server.playPopen is not None: if self.hugvey.cmd_server.playPopen is not None:
logger.debug('block recording {}' .format( self.hugvey.cmd_server.playPopen)) logger.debug('block recording {}' .format(
# multiply by 0 to disable audio recording while playback self.hugvey.cmd_server.playPopen))
f = audioop.mul(f, 2, 0) # multiply by 0 to disable audio recording while playback
f = audioop.mul(f, 2, 0)
self.loop.call_soon_threadsafe( self.voice_socket.send, f )
self.loop.call_soon_threadsafe(self.voice_socket.send, f)
except Exception as e: except Exception as e:
logger.warn("Error sending to {}".format(e)) logger.warn("Error sending to {}".format(e))
pass pass
@ -88,19 +94,20 @@ class VoiceServer(object):
frames_per_buffer=CHUNK, frames_per_buffer=CHUNK,
stream_callback=self.onBuffer, stream_callback=self.onBuffer,
input_device_index=self.get_input_idx() input_device_index=self.get_input_idx()
) )
while not self.stopped: while not self.stopped:
try: try:
address = "tcp://*:{}".format(self.voice_port) address = "tcp://*:{}".format(self.voice_port)
self.voice_socket = self.ctx.socket(zmq.PUB) self.voice_socket = self.ctx.socket(zmq.PUB)
self.voice_socket.bind(address) self.voice_socket.bind(address)
logger.info( "Waiting for voice connections on {}".format(address) ) logger.info(
"Waiting for voice connections on {}".format(address))
while not self.stopped: while not self.stopped:
await asyncio.sleep(1) await asyncio.sleep(1)
logger.info( "Stop recording & streaming") logger.info("Stop recording & streaming")
self.voice_socket.close() self.voice_socket.close()
# stop Recording # stop Recording
stream.stop_stream() stream.stop_stream()
@ -118,6 +125,7 @@ class VoiceServer(object):
future = loop.run_in_executor(None, self.start) future = loop.run_in_executor(None, self.start)
r = await future r = await future
class CommandHandler(object): class CommandHandler(object):
def __init__(self, hugvey_id, cmd_address, publish_address, file_address): def __init__(self, hugvey_id, cmd_address, publish_address, file_address):
self.eventQueue = [] self.eventQueue = []
@ -145,42 +153,58 @@ class CommandHandler(object):
if cmd['action'] == 'stop': if cmd['action'] == 'stop':
self.cmdPlay(cmd, cmd['id']) self.cmdPlay(cmd, cmd['id'])
def cmdPlay(self, cmd): def cmdPlay(self, cmd):
msgId= cmd['id'] msgId = cmd['id']
pitch = cmd['pitch'] if 'pitch' in cmd else 50 pitch = cmd['pitch'] if 'pitch' in cmd else 50
file = cmd['file'] if 'file' in cmd else None file = cmd['file'] if 'file' in cmd else None
text = cmd['msg'] if 'msg' in cmd else None text = cmd['msg'] if 'msg' in cmd else None
params = cmd['params'] if 'params' in cmd else {}
self.playingMsgId = msgId self.playingMsgId = msgId
if file is None and text is None: if file is None and text is None:
logger.critical("No file nor text given: {}".format(cmd)) logger.critical("No file nor text given: {}".format(cmd))
else: else:
if file is not None: if file is not None:
logger.info("Play: {}".format(file)) logger.info("Play: {}".format(file))
file = self.file_address + "/" + file file = self.file_address + "/" + file
logger.debug(['play', file]) # logger.debug(['play', file])
self.playPopen = subprocess.Popen(['play', file], stdout=subprocess.PIPE) playCmd = ['play', file]
for param, value in params.items():
if not value:
continue
playCmd.append(param)
print(param, value)
if value is True:
continue
playCmd.append(str(value))
logger.debug(playCmd)
self.playPopen = subprocess.Popen(
playCmd, stdout=subprocess.PIPE)
returnCode = self.playPopen.wait() returnCode = self.playPopen.wait()
logger.debug('finished') logger.debug('finished')
self.playPopen = None self.playPopen = None
else: else:
logger.info("Speak: {}".format(text)) logger.info("Speak: {}".format(text))
self.playPopen = subprocess.Popen(['espeak', '-p','{0}'.format(pitch), text], stdout=subprocess.PIPE) self.playPopen = subprocess.Popen(
['espeak', '-p', '{0}'.format(pitch), text], stdout=subprocess.PIPE)
returnCode = self.playPopen.wait() returnCode = self.playPopen.wait()
self.playPopen = None self.playPopen = None
if returnCode: if returnCode:
logger.warn("Had returncode on play: {}".format(returnCode)) logger.warn("Had returncode on play: {}".format(returnCode))
else: else:
logger.debug("Finished playback. Return code: {}".format(returnCode)) logger.debug(
"Finished playback. Return code: {}".format(returnCode))
self.playingMsgId = None self.playingMsgId = None
self.sendMessage({ self.sendMessage({
'event': 'playbackFinish', 'event': 'playbackFinish',
'msgId': msgId 'msgId': msgId
}) })
def cmdStop(self, msgId): def cmdStop(self, msgId):
if self.playPopen and self.playingMsgId == msgId: if self.playPopen and self.playingMsgId == msgId:
logger.info("Interrupting playback") logger.info("Interrupting playback")
@ -207,7 +231,6 @@ class CommandHandler(object):
s.connect(("185.66.250.60", 80)) s.connect(("185.66.250.60", 80))
return s.getsockname()[0] return s.getsockname()[0]
def sendMessage(self, msg): def sendMessage(self, msg):
self.eventQueue.append(msg) self.eventQueue.append(msg)
@ -216,7 +239,8 @@ class CommandHandler(object):
s.connect(self.cmd_address) s.connect(self.cmd_address)
topic = getTopic(self.hugvey_id) topic = getTopic(self.hugvey_id)
s.subscribe(topic) s.subscribe(topic)
logger.info("Subscribed to commands for {} on {}".format(topic, self.cmd_address)) logger.info("Subscribed to commands for {} on {}".format(
topic, self.cmd_address))
while True: while True:
hugvey_id, cmd = await zmqReceive(s) hugvey_id, cmd = await zmqReceive(s)
# print("GOGOG", hugvey_id, cmd) # print("GOGOG", hugvey_id, cmd)
@ -232,11 +256,11 @@ class CommandHandler(object):
logger.info("Publish on: {}".format(self.publish_address)) logger.info("Publish on: {}".format(self.publish_address))
# For some reason, sending only one message is lost, perhaps due # For some reason, sending only one message is lost, perhaps due
# to connect() rather than bind() ?? # to connect() rather than bind() ??
await asyncio.sleep(1) # wait for connection to be proper set await asyncio.sleep(1) # wait for connection to be proper set
self.showMyself() self.showMyself()
while True: while True:
for i in range(len(self.eventQueue)): for i in range(len(self.eventQueue)):
zmqSend(s, self.hugvey_id, self.eventQueue.pop(0)) zmqSend(s, self.hugvey_id, self.eventQueue.pop(0))
@ -245,9 +269,11 @@ class CommandHandler(object):
s.close() s.close()
class Hugvey(object): class Hugvey(object):
"""The Hugvey client, to be ran on the Raspberry Pi's """The Hugvey client, to be ran on the Raspberry Pi's
""" """
def __init__(self): def __init__(self):
self.id = self.getId() self.id = self.getId()
pass pass
@ -256,7 +282,7 @@ class Hugvey(object):
"""Get Hugvey ID from hostname""" """Get Hugvey ID from hostname"""
try: try:
h = socket.gethostname() h = socket.gethostname()
id = int(re.findall('\d+', h )[0]) id = int(re.findall('\d+', h)[0])
except Exception: except Exception:
logger.critical("No automatic ID, fall back to 1") logger.critical("No automatic ID, fall back to 1")
id = 1 id = 1
@ -273,24 +299,25 @@ class Hugvey(object):
def start(self): def start(self):
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
if self.config['voice']['play_device'] and 'alsaaudio' in sys.modules: if self.config['voice']['play_device'] and 'alsaaudio' in sys.modules:
alsaaudio.Mixer(self.config['voice']['play_device']).setvolume(self.config['voice']['play_volume']) alsaaudio.Mixer(self.config['voice']['play_device']).setvolume(
self.config['voice']['play_volume'])
self.cmd_server = CommandHandler( self.cmd_server = CommandHandler(
hugvey_id = self.id, hugvey_id=self.id,
cmd_address = self.config['events']['cmd_address'], cmd_address=self.config['events']['cmd_address'],
publish_address = self.config['events']['publish_address'], publish_address=self.config['events']['publish_address'],
file_address = self.config['voice']['file_address'] file_address=self.config['voice']['file_address']
) )
self.voice_server = VoiceServer( self.voice_server = VoiceServer(
loop = loop, loop=loop,
hugvey = self, hugvey=self,
voice_port = int(self.config['voice']['port']), voice_port=int(self.config['voice']['port']),
input_rate = int(self.config['voice']['input_rate']), input_rate=int(self.config['voice']['input_rate']),
input_name = self.config['voice']['input_name'], input_name=self.config['voice']['input_name'],
target_rate = int(self.config['voice']['target_rate']), target_rate=int(self.config['voice']['target_rate']),
) )
logger.info('start') logger.info('start')
# self.voice_server.asyncStart(loop) # self.voice_server.asyncStart(loop)
# loop.run_until_complete(self.voice_server.start()) # loop.run_until_complete(self.voice_server.start())

16
hugvey/story.py
View file

@ -38,6 +38,7 @@ class Message(object):
self.interruptCount = 0 self.interruptCount = 0
self.afterrunTime = 0. # the time after this message to allow for interrupts self.afterrunTime = 0. # the time after this message to allow for interrupts
self.finishTime = None # message can be finished without finished utterance (with instant replycontains) self.finishTime = None # message can be finished without finished utterance (with instant replycontains)
self.params = {}
self.variableValues = {} self.variableValues = {}
self.parseForVariables() self.parseForVariables()
@ -52,6 +53,8 @@ class Message(object):
if 'audio' in data: if 'audio' in data:
msg.audioFile = data['audio']['file'] msg.audioFile = data['audio']['file']
msg.setStory(story) msg.setStory(story)
if 'params' in data:
msg.params = data['params']
return msg return msg
def parseForVariables(self): def parseForVariables(self):
@ -90,7 +93,8 @@ class Message(object):
logger.debug(self.variables) logger.debug(self.variables)
for var in self.variables: for var in self.variables:
logger.debug(f"try replacing ${var} with {self.variableValues[var]} in {text}") logger.debug(f"try replacing ${var} with {self.variableValues[var]} in {text}")
text = text.replace('$'+var, self.variableValues[var]) replacement = self.variableValues[var] if (self.variableValues[var] is not None) else "nothing" #TODO: translate nothing to each language
text = text.replace('$'+var, replacement)
return text return text
def setReply(self, reply): def setReply(self, reply):
@ -114,6 +118,9 @@ class Message(object):
def getFinishedTime(self): def getFinishedTime(self):
return self.finishTime return self.finishTime
def getParams(self):
return self.params
def getLogSummary(self): def getLogSummary(self):
return { return {
@ -268,7 +275,7 @@ class Condition(object):
return False return False
logger.debug('Got match on {}'.format(self.vars['regex'])) logger.debug('Got match on {}'.format(self.vars['regex']))
if 'instantMatch' in self.vars and self.vars['instantMatch'] or not r.isSpeaking(): if ('instantMatch' in self.vars and self.vars['instantMatch']) or not r.isSpeaking():
# try to avoid setting variables for intermediate strings # try to avoid setting variables for intermediate strings
results = result.groupdict() results = result.groupdict()
for captureGroup in results: for captureGroup in results:
@ -540,6 +547,10 @@ class Story(object):
self.log = [] # all nodes/elements that are triggered self.log = [] # all nodes/elements that are triggered
self.currentReply = None self.currentReply = None
self.stats = {
'timeouts': 0,
}
for msg in self.getMessages(): for msg in self.getMessages():
pass pass
@ -708,6 +719,7 @@ class Story(object):
'action': 'play', 'action': 'play',
'file': await message.getAudioFilePath(), 'file': await message.getAudioFilePath(),
'id': message.id, 'id': message.id,
'params': message.getParams()
}) })
# 2019-02-22 temporary disable listening while playing audio: # 2019-02-22 temporary disable listening while playing audio:

14
www/css/styles.css
View file

@ -108,18 +108,16 @@ img.icon {
cursor: grabbing; } cursor: grabbing; }
#story svg#graph .beenHit circle { #story svg#graph .beenHit circle {
stroke: #0f0; stroke: #0f0;
stroke-width: 2px; } stroke-width: 25px; }
#story svg#graph line.beenHit { #story svg#graph line.beenHit {
stroke: #0f0; } stroke: #0f0; }
#story circle { #story circle {
cursor: pointer; cursor: pointer;
fill: #77618e; } fill: #77618e; }
#story .startMsg circle { #story .startMsg circle {
fill: lightseagreen; } fill: lightseagreen !important; }
#story .endMsg circle {
fill: lightslategray; }
#story .orphanedMsg { #story .orphanedMsg {
fill: lightcoral; } fill: lightcoral !important; }
#story text { #story text {
text-anchor: middle; text-anchor: middle;
font-size: 11pt; font-size: 11pt;
@ -132,7 +130,7 @@ img.icon {
font-weight: bold; } font-weight: bold; }
#story line { #story line {
marker-end: url("#arrowHead"); marker-end: url("#arrowHead");
stroke-width: 2px; stroke-width: 5px;
stroke: black; } stroke: black; }
#story line.link--noconditions { #story line.link--noconditions {
stroke-dasharray: 5 4; stroke-dasharray: 5 4;
@ -171,6 +169,8 @@ img.icon {
padding: 10px; padding: 10px;
margin-bottom: 10px; margin-bottom: 10px;
background: lightgray; } background: lightgray; }
#story #msg .msg__info.btn, #story #msg .directions > div.btn {
background: darkblue; }
#story #msg h1 { #story #msg h1 {
margin: 0; } margin: 0; }
#story #msg .msg__info .btn--delete-mss { #story #msg .msg__info .btn--delete-mss {
@ -190,7 +190,7 @@ img.icon {
#story #nodes g:hover circle, #story #nodes g:hover circle,
#story .selectedMsg circle { #story .selectedMsg circle {
stroke: lightgreen; stroke: lightgreen;
stroke-width: 8; } stroke-width: 27; }
#story .controlDown #nodes g:hover circle, #story .controlDown #nodes g:hover circle,
#story .secondaryMsg circle { #story .secondaryMsg circle {
stroke: lightgreen; stroke: lightgreen;

96
www/js/hugvey_console.js
View file

@ -292,6 +292,13 @@ class Graph {
startAttributes['checked'] = 'checked'; startAttributes['checked'] = 'checked';
} }
let params = {};
if(msg.hasOwnProperty('params')) {
params = msg['params'];
} else {
msg['params'] = {};
}
let audioSrcEl = crel('source', {'src': msg['audio'] ? msg['audio']['file'] : this.getAudioUrlForMsg(msg)}); let audioSrcEl = crel('source', {'src': msg['audio'] ? msg['audio']['file'] : this.getAudioUrlForMsg(msg)});
// console.log(msg['audio']); // console.log(msg['audio']);
let audioSpan = crel( let audioSpan = crel(
@ -374,16 +381,76 @@ class Graph {
} ) } )
), ),
crel( 'label', crel( 'label',
crel( 'span', { crel( 'span', {
"title": "The time after the reply in which one can still interrupt to continue speaking" "title": "The time after the reply in which one can still interrupt to continue speaking"
}, 'Afterrun time' ), }, 'Afterrun time' ),
crel( 'input', { crel( 'input', {
'name': msg['@id'] + '-afterrunTime', 'name': msg['@id'] + '-afterrunTime',
'value': msg['afterrunTime'], 'step': "0.1",
'type': 'number', 'value': msg['afterrunTime'],
'on': { 'type': 'number',
'change': this.getEditEventListener() 'on': {
} 'change': this.getEditEventListener()
}
} )
),
crel( 'label',
crel( 'span', {
"title": "Playback volume factor"
}, 'Volume factor' ),
crel( 'input', {
'name': msg['@id'] + '-params.vol',
'value': params.hasOwnProperty('vol') ? params['vol'] : 1,
'step': "0.1",
'type': 'number',
'on': {
'change': this.getEditEventListener()
}
} )
),
crel( 'label',
crel( 'span', {
"title": "Playback tempo factor"
}, 'Tempo factor' ),
crel( 'input', {
'name': msg['@id'] + '-params.tempo',
'value': params.hasOwnProperty('tempo') ? params['tempo'] : 1,
'step': "0.1",
'min': "0.1",
'type': 'number',
'on': {
'change': this.getEditEventListener()
}
} )
),
crel( 'label',
crel( 'span', {
"title": "Playback pitch factor"
}, 'Pitch factor' ),
crel( 'input', {
'name': msg['@id'] + '-params.pitch',
'value': params.hasOwnProperty('pitch') ? params['pitch'] : 0,
'step': "0.1",
'type': 'number',
'on': {
'change': this.getEditEventListener()
}
} )
),
// color for beter overview
crel( 'label',
crel( 'span', {
"title": "Color - for your eyes only"
}, 'Color' ),
crel( 'input', {
'name': msg['@id'] + '-color',
'value': msg.hasOwnProperty('color') ? msg['color'] : '#77618e',
'type': 'color',
'on': {
'change': this.getEditEventListener()
}
} ) } )
) )
); );
@ -863,7 +930,12 @@ class Graph {
createConnectedMsg(sourceMsg) { createConnectedMsg(sourceMsg) {
let newMsg = this.addMsg(); let newMsg = this.addMsg();
this.getNodeById(newMsg['@id']).y = this.getNodeById(sourceMsg['@id']).y this.getNodeById(newMsg['@id']).y = this.getNodeById(sourceMsg['@id']).y;
if(this.getNodeById(sourceMsg['@id']).hasOwnProperty('color')){
this.getNodeById(newMsg['@id']).color = this.getNodeById(sourceMsg['@id']).color
}
this.addDirection(sourceMsg, newMsg); this.addDirection(sourceMsg, newMsg);
this.build(); this.build();
@ -1127,6 +1199,8 @@ class Graph {
// .container(document.getElementById('container')) // .container(document.getElementById('container'))
); );
node.select('circle').attr('style', (d) => 'fill: ' + (d.hasOwnProperty('color') ? d['color'] : '#77618e'));
let link = this.linkG let link = this.linkG
.selectAll( "line" ) .selectAll( "line" )

16
www/scss/styles.scss
View file

@ -187,7 +187,7 @@ img.icon{
.beenHit{ .beenHit{
circle { circle {
stroke: #0f0; stroke: #0f0;
stroke-width: 2px; stroke-width: 25px;
} }
} }
line.beenHit { line.beenHit {
@ -200,14 +200,14 @@ img.icon{
fill: rgb(119, 97, 142); fill: rgb(119, 97, 142);
} }
.startMsg circle{ .startMsg circle{
fill: lightseagreen; fill: lightseagreen !important;
} }
.endMsg circle{ .endMsg circle{
fill: lightslategray; // fill: lightslategray !important;
} }
.orphanedMsg{ .orphanedMsg{
fill: lightcoral; fill: lightcoral !important;
} }
text{ text{
text-anchor: middle; text-anchor: middle;
@ -225,7 +225,7 @@ img.icon{
} }
line{ line{
marker-end: url('#arrowHead'); marker-end: url('#arrowHead');
stroke-width: 2px; stroke-width: 5px;
stroke: black; stroke: black;
&.link--noconditions{ &.link--noconditions{
@ -278,6 +278,10 @@ img.icon{
padding: 10px; padding: 10px;
margin-bottom: 10px; margin-bottom: 10px;
background:lightgray; background:lightgray;
&.btn{
background: darkblue;
}
} }
h1{ h1{
@ -315,7 +319,7 @@ img.icon{
#nodes g:hover circle, #nodes g:hover circle,
.selectedMsg circle { .selectedMsg circle {
stroke: lightgreen; stroke: lightgreen;
stroke-width: 8; stroke-width: 27;
} }
.controlDown #nodes g:hover circle, .controlDown #nodes g:hover circle,