Colors and SFX
This commit is contained in:
Ruben van de Ven
parent
a798b3b638
commit
1563f9d1af
129
hugvey/client.py
129
hugvey/client.py
|
|
@ -10,11 +10,12 @@ import yaml
|
|||
import zmq
|
||||
from zmq.asyncio import Context
|
||||
import sys
|
||||
from hugvey.communication import LOG_BS
|
||||
|
||||
try:
|
||||
import alsaaudio
|
||||
import alsaaudio
|
||||
except ImportError:
|
||||
print("No volume settings available")
|
||||
print("No volume settings available")
|
||||
|
||||
from .communication import zmqReceive, zmqSend, getTopic
|
||||
import subprocess
|
||||
|
|
@ -22,8 +23,10 @@ import subprocess
|
|||
|
||||
logger = logging.getLogger("client")
|
||||
|
||||
|
||||
class VoiceServer(object):
|
||||
"""A UDP server, providing mic data at 16 kHz"""
|
||||
|
||||
def __init__(self, loop, hugvey, voice_port: int, input_rate: int, input_name: str = None, target_rate: int = 16000):
|
||||
self.voice_port = voice_port
|
||||
self.input_rate = input_rate
|
||||
|
|
@ -45,28 +48,31 @@ class VoiceServer(object):
|
|||
dev = self.p.get_device_info_by_index(i)
|
||||
if input_device_idx is None and dev['maxInputChannels'] > 0:
|
||||
if (self.input_name and self.input_name in dev['name']) or \
|
||||
(not self.input_name and dev['name'] != 'default'):
|
||||
(not self.input_name and dev['name'] != 'default'):
|
||||
input_device_idx = dev['index']
|
||||
logger.info("Use device {0}: {1}".format(dev['index'],dev['name']))
|
||||
logger.debug("{} {:0d} {}".format("* " if input_device_idx == i else "- ", i, dev['name']))
|
||||
logger.info("Use device {0}: {1}".format(
|
||||
dev['index'], dev['name']))
|
||||
logger.debug("{} {:0d} {}".format(
|
||||
"* " if input_device_idx == i else "- ", i, dev['name']))
|
||||
return input_device_idx
|
||||
|
||||
|
||||
def onBuffer(self, in_data, frame_count, time_info, status):
|
||||
if self.input_rate == self.target_rate:
|
||||
f = in_data
|
||||
else:
|
||||
# chunk 4096, with 2 bytes per frame gives len(in_data) of 8192
|
||||
# rate converted 44k1 -> 16k gives len(f) == 2972 (16/44.1 * 8192)
|
||||
f, self.laststate = audioop.ratecv(in_data, 2, 1, self.input_rate, self.target_rate, self.laststate)
|
||||
|
||||
f, self.laststate = audioop.ratecv(
|
||||
in_data, 2, 1, self.input_rate, self.target_rate, self.laststate)
|
||||
|
||||
try:
|
||||
if self.hugvey.cmd_server.playPopen is not None:
|
||||
logger.debug('block recording {}' .format( self.hugvey.cmd_server.playPopen))
|
||||
# multiply by 0 to disable audio recording while playback
|
||||
f = audioop.mul(f, 2, 0)
|
||||
|
||||
self.loop.call_soon_threadsafe( self.voice_socket.send, f )
|
||||
if self.hugvey.cmd_server.playPopen is not None:
|
||||
logger.debug('block recording {}' .format(
|
||||
self.hugvey.cmd_server.playPopen))
|
||||
# multiply by 0 to disable audio recording while playback
|
||||
f = audioop.mul(f, 2, 0)
|
||||
|
||||
self.loop.call_soon_threadsafe(self.voice_socket.send, f)
|
||||
except Exception as e:
|
||||
logger.warn("Error sending to {}".format(e))
|
||||
pass
|
||||
|
|
@ -88,19 +94,20 @@ class VoiceServer(object):
|
|||
frames_per_buffer=CHUNK,
|
||||
stream_callback=self.onBuffer,
|
||||
input_device_index=self.get_input_idx()
|
||||
)
|
||||
)
|
||||
|
||||
while not self.stopped:
|
||||
try:
|
||||
address = "tcp://*:{}".format(self.voice_port)
|
||||
self.voice_socket = self.ctx.socket(zmq.PUB)
|
||||
self.voice_socket.bind(address)
|
||||
|
||||
logger.info( "Waiting for voice connections on {}".format(address) )
|
||||
|
||||
logger.info(
|
||||
"Waiting for voice connections on {}".format(address))
|
||||
while not self.stopped:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info( "Stop recording & streaming")
|
||||
logger.info("Stop recording & streaming")
|
||||
self.voice_socket.close()
|
||||
# stop Recording
|
||||
stream.stop_stream()
|
||||
|
|
@ -118,6 +125,7 @@ class VoiceServer(object):
|
|||
future = loop.run_in_executor(None, self.start)
|
||||
r = await future
|
||||
|
||||
|
||||
class CommandHandler(object):
|
||||
def __init__(self, hugvey_id, cmd_address, publish_address, file_address):
|
||||
self.eventQueue = []
|
||||
|
|
@ -145,42 +153,58 @@ class CommandHandler(object):
|
|||
if cmd['action'] == 'stop':
|
||||
self.cmdPlay(cmd, cmd['id'])
|
||||
|
||||
|
||||
def cmdPlay(self, cmd):
|
||||
msgId= cmd['id']
|
||||
msgId = cmd['id']
|
||||
pitch = cmd['pitch'] if 'pitch' in cmd else 50
|
||||
file = cmd['file'] if 'file' in cmd else None
|
||||
text = cmd['msg'] if 'msg' in cmd else None
|
||||
params = cmd['params'] if 'params' in cmd else {}
|
||||
self.playingMsgId = msgId
|
||||
|
||||
|
||||
if file is None and text is None:
|
||||
logger.critical("No file nor text given: {}".format(cmd))
|
||||
else:
|
||||
if file is not None:
|
||||
logger.info("Play: {}".format(file))
|
||||
file = self.file_address + "/" + file
|
||||
logger.debug(['play', file])
|
||||
self.playPopen = subprocess.Popen(['play', file], stdout=subprocess.PIPE)
|
||||
# logger.debug(['play', file])
|
||||
playCmd = ['play', file]
|
||||
|
||||
for param, value in params.items():
|
||||
if not value:
|
||||
continue
|
||||
playCmd.append(param)
|
||||
print(param, value)
|
||||
if value is True:
|
||||
continue
|
||||
playCmd.append(str(value))
|
||||
|
||||
logger.debug(playCmd)
|
||||
self.playPopen = subprocess.Popen(
|
||||
playCmd, stdout=subprocess.PIPE)
|
||||
|
||||
returnCode = self.playPopen.wait()
|
||||
logger.debug('finished')
|
||||
self.playPopen = None
|
||||
else:
|
||||
else:
|
||||
logger.info("Speak: {}".format(text))
|
||||
self.playPopen = subprocess.Popen(['espeak', '-p','{0}'.format(pitch), text], stdout=subprocess.PIPE)
|
||||
self.playPopen = subprocess.Popen(
|
||||
['espeak', '-p', '{0}'.format(pitch), text], stdout=subprocess.PIPE)
|
||||
returnCode = self.playPopen.wait()
|
||||
self.playPopen = None
|
||||
|
||||
|
||||
if returnCode:
|
||||
logger.warn("Had returncode on play: {}".format(returnCode))
|
||||
else:
|
||||
logger.debug("Finished playback. Return code: {}".format(returnCode))
|
||||
|
||||
logger.debug(
|
||||
"Finished playback. Return code: {}".format(returnCode))
|
||||
|
||||
self.playingMsgId = None
|
||||
self.sendMessage({
|
||||
'event': 'playbackFinish',
|
||||
'msgId': msgId
|
||||
})
|
||||
|
||||
|
||||
def cmdStop(self, msgId):
|
||||
if self.playPopen and self.playingMsgId == msgId:
|
||||
logger.info("Interrupting playback")
|
||||
|
|
@ -207,7 +231,6 @@ class CommandHandler(object):
|
|||
s.connect(("185.66.250.60", 80))
|
||||
return s.getsockname()[0]
|
||||
|
||||
|
||||
def sendMessage(self, msg):
|
||||
self.eventQueue.append(msg)
|
||||
|
||||
|
|
@ -216,7 +239,8 @@ class CommandHandler(object):
|
|||
s.connect(self.cmd_address)
|
||||
topic = getTopic(self.hugvey_id)
|
||||
s.subscribe(topic)
|
||||
logger.info("Subscribed to commands for {} on {}".format(topic, self.cmd_address))
|
||||
logger.info("Subscribed to commands for {} on {}".format(
|
||||
topic, self.cmd_address))
|
||||
while True:
|
||||
hugvey_id, cmd = await zmqReceive(s)
|
||||
# print("GOGOG", hugvey_id, cmd)
|
||||
|
|
@ -232,11 +256,11 @@ class CommandHandler(object):
|
|||
logger.info("Publish on: {}".format(self.publish_address))
|
||||
# For some reason, sending only one message is lost, perhaps due
|
||||
# to connect() rather than bind() ??
|
||||
|
||||
await asyncio.sleep(1) # wait for connection to be proper set
|
||||
|
||||
|
||||
await asyncio.sleep(1) # wait for connection to be proper set
|
||||
|
||||
self.showMyself()
|
||||
|
||||
|
||||
while True:
|
||||
for i in range(len(self.eventQueue)):
|
||||
zmqSend(s, self.hugvey_id, self.eventQueue.pop(0))
|
||||
|
|
@ -245,9 +269,11 @@ class CommandHandler(object):
|
|||
|
||||
s.close()
|
||||
|
||||
|
||||
class Hugvey(object):
|
||||
"""The Hugvey client, to be ran on the Raspberry Pi's
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.id = self.getId()
|
||||
pass
|
||||
|
|
@ -256,7 +282,7 @@ class Hugvey(object):
|
|||
"""Get Hugvey ID from hostname"""
|
||||
try:
|
||||
h = socket.gethostname()
|
||||
id = int(re.findall('\d+', h )[0])
|
||||
id = int(re.findall('\d+', h)[0])
|
||||
except Exception:
|
||||
logger.critical("No automatic ID, fall back to 1")
|
||||
id = 1
|
||||
|
|
@ -273,24 +299,25 @@ class Hugvey(object):
|
|||
|
||||
def start(self):
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
|
||||
if self.config['voice']['play_device'] and 'alsaaudio' in sys.modules:
|
||||
alsaaudio.Mixer(self.config['voice']['play_device']).setvolume(self.config['voice']['play_volume'])
|
||||
|
||||
alsaaudio.Mixer(self.config['voice']['play_device']).setvolume(
|
||||
self.config['voice']['play_volume'])
|
||||
|
||||
self.cmd_server = CommandHandler(
|
||||
hugvey_id = self.id,
|
||||
cmd_address = self.config['events']['cmd_address'],
|
||||
publish_address = self.config['events']['publish_address'],
|
||||
file_address = self.config['voice']['file_address']
|
||||
)
|
||||
hugvey_id=self.id,
|
||||
cmd_address=self.config['events']['cmd_address'],
|
||||
publish_address=self.config['events']['publish_address'],
|
||||
file_address=self.config['voice']['file_address']
|
||||
)
|
||||
self.voice_server = VoiceServer(
|
||||
loop = loop,
|
||||
hugvey = self,
|
||||
voice_port = int(self.config['voice']['port']),
|
||||
input_rate = int(self.config['voice']['input_rate']),
|
||||
input_name = self.config['voice']['input_name'],
|
||||
target_rate = int(self.config['voice']['target_rate']),
|
||||
)
|
||||
loop=loop,
|
||||
hugvey=self,
|
||||
voice_port=int(self.config['voice']['port']),
|
||||
input_rate=int(self.config['voice']['input_rate']),
|
||||
input_name=self.config['voice']['input_name'],
|
||||
target_rate=int(self.config['voice']['target_rate']),
|
||||
)
|
||||
logger.info('start')
|
||||
# self.voice_server.asyncStart(loop)
|
||||
# loop.run_until_complete(self.voice_server.start())
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ class Message(object):
|
|||
self.interruptCount = 0
|
||||
self.afterrunTime = 0. # the time after this message to allow for interrupts
|
||||
self.finishTime = None # message can be finished without finished utterance (with instant replycontains)
|
||||
self.params = {}
|
||||
self.variableValues = {}
|
||||
self.parseForVariables()
|
||||
|
||||
|
|
@ -52,6 +53,8 @@ class Message(object):
|
|||
if 'audio' in data:
|
||||
msg.audioFile = data['audio']['file']
|
||||
msg.setStory(story)
|
||||
if 'params' in data:
|
||||
msg.params = data['params']
|
||||
return msg
|
||||
|
||||
def parseForVariables(self):
|
||||
|
|
@ -90,7 +93,8 @@ class Message(object):
|
|||
logger.debug(self.variables)
|
||||
for var in self.variables:
|
||||
logger.debug(f"try replacing ${var} with {self.variableValues[var]} in {text}")
|
||||
text = text.replace('$'+var, self.variableValues[var])
|
||||
replacement = self.variableValues[var] if (self.variableValues[var] is not None) else "nothing" #TODO: translate nothing to each language
|
||||
text = text.replace('$'+var, replacement)
|
||||
return text
|
||||
|
||||
def setReply(self, reply):
|
||||
|
|
@ -114,6 +118,9 @@ class Message(object):
|
|||
|
||||
def getFinishedTime(self):
|
||||
return self.finishTime
|
||||
|
||||
def getParams(self):
|
||||
return self.params
|
||||
|
||||
def getLogSummary(self):
|
||||
return {
|
||||
|
|
@ -268,7 +275,7 @@ class Condition(object):
|
|||
return False
|
||||
logger.debug('Got match on {}'.format(self.vars['regex']))
|
||||
|
||||
if 'instantMatch' in self.vars and self.vars['instantMatch'] or not r.isSpeaking():
|
||||
if ('instantMatch' in self.vars and self.vars['instantMatch']) or not r.isSpeaking():
|
||||
# try to avoid setting variables for intermediate strings
|
||||
results = result.groupdict()
|
||||
for captureGroup in results:
|
||||
|
|
@ -540,6 +547,10 @@ class Story(object):
|
|||
self.log = [] # all nodes/elements that are triggered
|
||||
self.currentReply = None
|
||||
|
||||
self.stats = {
|
||||
'timeouts': 0,
|
||||
}
|
||||
|
||||
for msg in self.getMessages():
|
||||
pass
|
||||
|
||||
|
|
@ -708,6 +719,7 @@ class Story(object):
|
|||
'action': 'play',
|
||||
'file': await message.getAudioFilePath(),
|
||||
'id': message.id,
|
||||
'params': message.getParams()
|
||||
})
|
||||
|
||||
# 2019-02-22 temporary disable listening while playing audio:
|
||||
|
|
|
|||
|
|
@ -108,18 +108,16 @@ img.icon {
|
|||
cursor: grabbing; }
|
||||
#story svg#graph .beenHit circle {
|
||||
stroke: #0f0;
|
||||
stroke-width: 2px; }
|
||||
stroke-width: 25px; }
|
||||
#story svg#graph line.beenHit {
|
||||
stroke: #0f0; }
|
||||
#story circle {
|
||||
cursor: pointer;
|
||||
fill: #77618e; }
|
||||
#story .startMsg circle {
|
||||
fill: lightseagreen; }
|
||||
#story .endMsg circle {
|
||||
fill: lightslategray; }
|
||||
fill: lightseagreen !important; }
|
||||
#story .orphanedMsg {
|
||||
fill: lightcoral; }
|
||||
fill: lightcoral !important; }
|
||||
#story text {
|
||||
text-anchor: middle;
|
||||
font-size: 11pt;
|
||||
|
|
@ -132,7 +130,7 @@ img.icon {
|
|||
font-weight: bold; }
|
||||
#story line {
|
||||
marker-end: url("#arrowHead");
|
||||
stroke-width: 2px;
|
||||
stroke-width: 5px;
|
||||
stroke: black; }
|
||||
#story line.link--noconditions {
|
||||
stroke-dasharray: 5 4;
|
||||
|
|
@ -171,6 +169,8 @@ img.icon {
|
|||
padding: 10px;
|
||||
margin-bottom: 10px;
|
||||
background: lightgray; }
|
||||
#story #msg .msg__info.btn, #story #msg .directions > div.btn {
|
||||
background: darkblue; }
|
||||
#story #msg h1 {
|
||||
margin: 0; }
|
||||
#story #msg .msg__info .btn--delete-mss {
|
||||
|
|
@ -190,7 +190,7 @@ img.icon {
|
|||
#story #nodes g:hover circle,
|
||||
#story .selectedMsg circle {
|
||||
stroke: lightgreen;
|
||||
stroke-width: 8; }
|
||||
stroke-width: 27; }
|
||||
#story .controlDown #nodes g:hover circle,
|
||||
#story .secondaryMsg circle {
|
||||
stroke: lightgreen;
|
||||
|
|
|
|||
|
|
@ -292,6 +292,13 @@ class Graph {
|
|||
startAttributes['checked'] = 'checked';
|
||||
}
|
||||
|
||||
let params = {};
|
||||
if(msg.hasOwnProperty('params')) {
|
||||
params = msg['params'];
|
||||
} else {
|
||||
msg['params'] = {};
|
||||
}
|
||||
|
||||
let audioSrcEl = crel('source', {'src': msg['audio'] ? msg['audio']['file'] : this.getAudioUrlForMsg(msg)});
|
||||
// console.log(msg['audio']);
|
||||
let audioSpan = crel(
|
||||
|
|
@ -374,16 +381,76 @@ class Graph {
|
|||
} )
|
||||
),
|
||||
crel( 'label',
|
||||
crel( 'span', {
|
||||
"title": "The time after the reply in which one can still interrupt to continue speaking"
|
||||
}, 'Afterrun time' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-afterrunTime',
|
||||
'value': msg['afterrunTime'],
|
||||
'type': 'number',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
crel( 'span', {
|
||||
"title": "The time after the reply in which one can still interrupt to continue speaking"
|
||||
}, 'Afterrun time' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-afterrunTime',
|
||||
'step': "0.1",
|
||||
'value': msg['afterrunTime'],
|
||||
'type': 'number',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
} )
|
||||
),
|
||||
crel( 'label',
|
||||
crel( 'span', {
|
||||
"title": "Playback volume factor"
|
||||
}, 'Volume factor' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-params.vol',
|
||||
'value': params.hasOwnProperty('vol') ? params['vol'] : 1,
|
||||
'step': "0.1",
|
||||
'type': 'number',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
} )
|
||||
),
|
||||
crel( 'label',
|
||||
crel( 'span', {
|
||||
"title": "Playback tempo factor"
|
||||
}, 'Tempo factor' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-params.tempo',
|
||||
'value': params.hasOwnProperty('tempo') ? params['tempo'] : 1,
|
||||
'step': "0.1",
|
||||
'min': "0.1",
|
||||
'type': 'number',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
} )
|
||||
),
|
||||
crel( 'label',
|
||||
crel( 'span', {
|
||||
"title": "Playback pitch factor"
|
||||
}, 'Pitch factor' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-params.pitch',
|
||||
'value': params.hasOwnProperty('pitch') ? params['pitch'] : 0,
|
||||
'step': "0.1",
|
||||
'type': 'number',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
} )
|
||||
),
|
||||
|
||||
// color for beter overview
|
||||
|
||||
crel( 'label',
|
||||
crel( 'span', {
|
||||
"title": "Color - for your eyes only"
|
||||
}, 'Color' ),
|
||||
crel( 'input', {
|
||||
'name': msg['@id'] + '-color',
|
||||
'value': msg.hasOwnProperty('color') ? msg['color'] : '#77618e',
|
||||
'type': 'color',
|
||||
'on': {
|
||||
'change': this.getEditEventListener()
|
||||
}
|
||||
} )
|
||||
)
|
||||
);
|
||||
|
|
@ -863,7 +930,12 @@ class Graph {
|
|||
|
||||
createConnectedMsg(sourceMsg) {
|
||||
let newMsg = this.addMsg();
|
||||
this.getNodeById(newMsg['@id']).y = this.getNodeById(sourceMsg['@id']).y
|
||||
this.getNodeById(newMsg['@id']).y = this.getNodeById(sourceMsg['@id']).y;
|
||||
|
||||
if(this.getNodeById(sourceMsg['@id']).hasOwnProperty('color')){
|
||||
this.getNodeById(newMsg['@id']).color = this.getNodeById(sourceMsg['@id']).color
|
||||
}
|
||||
|
||||
this.addDirection(sourceMsg, newMsg);
|
||||
this.build();
|
||||
|
||||
|
|
@ -1127,6 +1199,8 @@ class Graph {
|
|||
// .container(document.getElementById('container'))
|
||||
|
||||
);
|
||||
|
||||
node.select('circle').attr('style', (d) => 'fill: ' + (d.hasOwnProperty('color') ? d['color'] : '#77618e'));
|
||||
|
||||
let link = this.linkG
|
||||
.selectAll( "line" )
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ img.icon{
|
|||
.beenHit{
|
||||
circle {
|
||||
stroke: #0f0;
|
||||
stroke-width: 2px;
|
||||
stroke-width: 25px;
|
||||
}
|
||||
}
|
||||
line.beenHit {
|
||||
|
|
@ -200,14 +200,14 @@ img.icon{
|
|||
fill: rgb(119, 97, 142);
|
||||
}
|
||||
.startMsg circle{
|
||||
fill: lightseagreen;
|
||||
fill: lightseagreen !important;
|
||||
}
|
||||
.endMsg circle{
|
||||
fill: lightslategray;
|
||||
// fill: lightslategray !important;
|
||||
}
|
||||
|
||||
.orphanedMsg{
|
||||
fill: lightcoral;
|
||||
fill: lightcoral !important;
|
||||
}
|
||||
text{
|
||||
text-anchor: middle;
|
||||
|
|
@ -225,7 +225,7 @@ img.icon{
|
|||
}
|
||||
line{
|
||||
marker-end: url('#arrowHead');
|
||||
stroke-width: 2px;
|
||||
stroke-width: 5px;
|
||||
stroke: black;
|
||||
|
||||
&.link--noconditions{
|
||||
|
|
@ -278,6 +278,10 @@ img.icon{
|
|||
padding: 10px;
|
||||
margin-bottom: 10px;
|
||||
background:lightgray;
|
||||
|
||||
&.btn{
|
||||
background: darkblue;
|
||||
}
|
||||
}
|
||||
|
||||
h1{
|
||||
|
|
@ -315,7 +319,7 @@ img.icon{
|
|||
#nodes g:hover circle,
|
||||
.selectedMsg circle {
|
||||
stroke: lightgreen;
|
||||
stroke-width: 8;
|
||||
stroke-width: 27;
|
||||
}
|
||||
|
||||
.controlDown #nodes g:hover circle,
|
||||
|
|
|
|||
Loading…
Reference in New Issue