spacecruft
/
nopenpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

soundd (#21619) 

* refactor alerts

* doesn't go here anymore

* soudd

* handle controls unresponsive

* same check

* fix path

* update sound test

* nice

* fix c2

* add script

* update tests

Co-authored-by: Comma Device <device@comma.ai>
pull/21633/head
Adeeb Shihadeh 2021-07-16 17:30:00 -07:00 committed by GitHub
parent 0f93cb12ac
commit 12948e661a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 169 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -42,6 +42,7 @@ selfdrive/logcatd/logcatd
selfdrive/mapd/default_speeds_by_region.json
selfdrive/proclogd/proclogd
selfdrive/ui/_ui
selfdrive/ui/_soundd
selfdrive/test/longitudinal_maneuvers/out
selfdrive/visiond/visiond
selfdrive/loggerd/loggerd

2
Jenkinsfile vendored
View File

@ -152,7 +152,7 @@ pipeline {
phone_steps("eon", [
["build", "cd selfdrive/manager && ./build.py"],
["test athena", "nosetests -s selfdrive/athena/tests/test_athenad_old.py"],
["test sounds", "nosetests -s selfdrive/test/test_sounds.py"],
["test sounds", "nosetests -s selfdrive/ui/tests/test_sounds.py"],
["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"],
["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
["test encoder", "python selfdrive/loggerd/tests/test_encoder.py"],

1
selfdrive/manager/process_config.py
View File

@ -18,6 +18,7 @@ procs = [
NativeProcess("sensord", "selfdrive/sensord", ["./sensord"], enabled=not PC, persistent=EON, sigkill=EON),
NativeProcess("ubloxd", "selfdrive/locationd", ["./ubloxd"], enabled=(not PC or WEBCAM)),
NativeProcess("ui", "selfdrive/ui", ["./ui"], persistent=True, watchdog_max_dt=(10 if TICI else None)),
NativeProcess("soundd", "selfdrive/ui", ["./soundd"]),
NativeProcess("locationd", "selfdrive/locationd", ["./locationd"]),
PythonProcess("calibrationd", "selfdrive.locationd.calibrationd"),
PythonProcess("controlsd", "selfdrive.controls.controlsd"),

5
selfdrive/test/test_onroad.py
View File

@ -34,6 +34,7 @@ PROCS = {
"./_dmonitoringmodeld": 2.67,
"selfdrive.thermald.thermald": 2.41,
"selfdrive.locationd.calibrationd": 2.0,
"./soundd": 2.0,
"selfdrive.monitoring.dmonitoringd": 1.90,
"./proclogd": 1.54,
"selfdrive.logmessaged": 0.2,
@ -48,6 +49,7 @@ if TICI:
"./loggerd": 60.0,
"selfdrive.controls.controlsd": 26.0,
"./camerad": 25.0,
"./_ui": 21.0,
"selfdrive.controls.plannerd": 12.0,
"selfdrive.locationd.paramsd": 5.0,
"./_dmonitoringmodeld": 10.0,
@ -74,9 +76,6 @@ def check_cpu_usage(first_proc, last_proc):
cpu_time = cputime_total(last) - cputime_total(first)
cpu_usage = cpu_time / dt * 100.
if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
# TODO: fix high CPU when playing sounds constantly in UI
if proc_name == "./_ui" and cpu_usage < 50.:
continue
result += f"Warning {proc_name} using more CPU than normal\n"
r = False
elif cpu_usage < min(normal_cpu_usage * 0.65, max(normal_cpu_usage - 1.0, 0.0)):

3
selfdrive/ui/SConscript
View File

@ -39,6 +39,9 @@ elif maps:
widgets = qt_env.Library("qt_widgets", widgets_src, LIBS=base_libs)
qt_libs = [widgets] + base_libs
# build soundd
qt_env.Program("_soundd", "soundd.cc", LIBS=base_libs)
# spinner and text window
qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs)
qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=qt_libs)

4
selfdrive/ui/main.cc
View File

@ -1,3 +1,5 @@
#include <sys/resource.h>
#include <QApplication>
#include <QSslConfiguration>
@ -7,6 +9,8 @@
#include "selfdrive/ui/qt/window.h"
int main(int argc, char *argv[]) {
setpriority(PRIO_PROCESS, 0, -20);
qInstallMessageHandler(swagLogMessageHandler);
initApp();

102
selfdrive/ui/qt/onroad.cc
View File

@ -67,45 +67,23 @@ void OnroadWindow::offroadTransition(bool offroad) {
// ***** onroad widgets *****
OnroadAlerts::OnroadAlerts(QWidget *parent) : QWidget(parent) {
std::tuple<AudibleAlert, QString, bool> sound_list[] = {
{AudibleAlert::CHIME_DISENGAGE, "../assets/sounds/disengaged.wav", false},
{AudibleAlert::CHIME_ENGAGE, "../assets/sounds/engaged.wav", false},
{AudibleAlert::CHIME_WARNING1, "../assets/sounds/warning_1.wav", false},
{AudibleAlert::CHIME_WARNING2, "../assets/sounds/warning_2.wav", false},
{AudibleAlert::CHIME_WARNING2_REPEAT, "../assets/sounds/warning_2.wav", true},
{AudibleAlert::CHIME_WARNING_REPEAT, "../assets/sounds/warning_repeat.wav", true},
{AudibleAlert::CHIME_ERROR, "../assets/sounds/error.wav", false},
{AudibleAlert::CHIME_PROMPT, "../assets/sounds/error.wav", false}};
for (auto &[alert, fn, loops] : sound_list) {
sounds[alert].first.setSource(QUrl::fromLocalFile(fn));
sounds[alert].second = loops ? QSoundEffect::Infinite : 0;
}
}
void OnroadAlerts::updateState(const UIState &s) {
SubMaster &sm = *(s.sm);
if (sm.updated("carState")) {
// scale volume with speed
volume = util::map_val(sm["carState"].getCarState().getVEgo(), 0.f, 20.f,
Hardware::MIN_VOLUME, Hardware::MAX_VOLUME);
}
if (sm["deviceState"].getDeviceState().getStarted()) {
if (sm.updated("controlsState")) {
const cereal::ControlsState::Reader &cs = sm["controlsState"].getControlsState();
updateAlert(QString::fromStdString(cs.getAlertText1()), QString::fromStdString(cs.getAlertText2()),
cs.getAlertBlinkingRate(), cs.getAlertType(), cs.getAlertSize(), cs.getAlertSound());
} else if ((sm.frame - s.scene.started_frame) > 10 * UI_FREQ) {
updateAlert({QString::fromStdString(cs.getAlertText1()),
QString::fromStdString(cs.getAlertText2()),
QString::fromStdString(cs.getAlertType()),
cs.getAlertSize(), cs.getAlertSound()});
} else if ((sm.frame - s.scene.started_frame) > 5 * UI_FREQ) {
// Handle controls timeout
if (sm.rcv_frame("controlsState") < s.scene.started_frame) {
// car is started, but controlsState hasn't been seen at all
updateAlert("openpilot Unavailable", "Waiting for controls to start", 0,
"controlsWaiting", cereal::ControlsState::AlertSize::MID, AudibleAlert::NONE);
} else if ((sm.frame - sm.rcv_frame("controlsState")) > 5 * UI_FREQ) {
updateAlert(CONTROLS_WAITING_ALERT);
} else if ((nanos_since_boot() - sm.rcv_time("controlsState")) / 1e9 > CONTROLS_TIMEOUT) {
// car is started, but controls is lagging or died
updateAlert("TAKE CONTROL IMMEDIATELY", "Controls Unresponsive", 0,
"controlsUnresponsive", cereal::ControlsState::AlertSize::FULL, AudibleAlert::CHIME_WARNING_REPEAT);
updateAlert(CONTROLS_UNRESPONSIVE_ALERT);
// TODO: clean this up once Qt handles the border
QUIState::ui_state.status = STATUS_ALERT;
@ -113,54 +91,22 @@ void OnroadAlerts::updateState(const UIState &s) {
}
}
// TODO: add blinking back if performant
//float alpha = 0.375 * cos((millis_since_boot() / 1000) * 2 * M_PI * blinking_rate) + 0.625;
bg = bg_colors[s.status];
}
void OnroadAlerts::offroadTransition(bool offroad) {
updateAlert("", "", 0, "", cereal::ControlsState::AlertSize::NONE, AudibleAlert::NONE);
updateAlert({});
}
void OnroadAlerts::updateAlert(const QString &t1, const QString &t2, float blink_rate,
const std::string &type, cereal::ControlsState::AlertSize size, AudibleAlert sound) {
if (alert_type.compare(type) == 0 && text1.compare(t1) == 0 && text2.compare(t2) == 0) {
return;
}
stopSounds();
if (sound != AudibleAlert::NONE) {
playSound(sound);
}
text1 = t1;
text2 = t2;
alert_type = type;
alert_size = size;
blinking_rate = blink_rate;
update();
}
void OnroadAlerts::playSound(AudibleAlert alert) {
auto &[sound, loops] = sounds[alert];
sound.setLoopCount(loops);
sound.setVolume(volume);
sound.play();
}
void OnroadAlerts::stopSounds() {
for (auto &kv : sounds) {
// Only stop repeating sounds
auto &[sound, loops] = kv.second;
if (sound.loopsRemaining() == QSoundEffect::Infinite) {
sound.stop();
}
void OnroadAlerts::updateAlert(Alert a) {
if (!alert.equal(a)) {
alert = a;
update();
}
}
void OnroadAlerts::paintEvent(QPaintEvent *event) {
if (alert_size == cereal::ControlsState::AlertSize::NONE) {
if (alert.size == cereal::ControlsState::AlertSize::NONE) {
return;
}
static std::map<cereal::ControlsState::AlertSize, const int> alert_sizes = {
@ -168,7 +114,7 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) {
{cereal::ControlsState::AlertSize::MID, 420},
{cereal::ControlsState::AlertSize::FULL, height()},
};
int h = alert_sizes[alert_size];
int h = alert_sizes[alert.size];
QRect r = QRect(0, height() - h, width(), h);
QPainter p(this);
@ -196,20 +142,20 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) {
const QPoint c = r.center();
p.setPen(QColor(0xff, 0xff, 0xff));
p.setRenderHint(QPainter::TextAntialiasing);
if (alert_size == cereal::ControlsState::AlertSize::SMALL) {
if (alert.size == cereal::ControlsState::AlertSize::SMALL) {
configFont(p, "Open Sans", 74, "SemiBold");
p.drawText(r, Qt::AlignCenter, text1);
} else if (alert_size == cereal::ControlsState::AlertSize::MID) {
p.drawText(r, Qt::AlignCenter, alert.text1);
} else if (alert.size == cereal::ControlsState::AlertSize::MID) {
configFont(p, "Open Sans", 88, "Bold");
p.drawText(QRect(0, c.y() - 125, width(), 150), Qt::AlignHCenter | Qt::AlignTop, text1);
p.drawText(QRect(0, c.y() - 125, width(), 150), Qt::AlignHCenter | Qt::AlignTop, alert.text1);
configFont(p, "Open Sans", 66, "Regular");
p.drawText(QRect(0, c.y() + 21, width(), 90), Qt::AlignHCenter, text2);
} else if (alert_size == cereal::ControlsState::AlertSize::FULL) {
bool l = text1.length() > 15;
p.drawText(QRect(0, c.y() + 21, width(), 90), Qt::AlignHCenter, alert.text2);
} else if (alert.size == cereal::ControlsState::AlertSize::FULL) {
bool l = alert.text1.length() > 15;
configFont(p, "Open Sans", l ? 132 : 177, "Bold");
p.drawText(QRect(0, r.y() + (l ? 240 : 270), width(), 600), Qt::AlignHCenter | Qt::TextWordWrap, text1);
p.drawText(QRect(0, r.y() + (l ? 240 : 270), width(), 600), Qt::AlignHCenter | Qt::TextWordWrap, alert.text1);
configFont(p, "Open Sans", 88, "Regular");
p.drawText(QRect(0, r.height() - (l ? 361 : 420), width(), 300), Qt::AlignHCenter | Qt::TextWordWrap, text2);
p.drawText(QRect(0, r.height() - (l ? 361 : 420), width(), 300), Qt::AlignHCenter | Qt::TextWordWrap, alert.text2);
}
}

20
selfdrive/ui/qt/onroad.h
View File

@ -1,15 +1,11 @@
#pragma once
#include <map>
#include <QOpenGLFunctions>
#include <QOpenGLWidget>
#include <QSoundEffect>
#include <QStackedLayout>
#include <QWidget>
#include "cereal/gen/cpp/log.capnp.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/ui/qt/qt_window.h"
#include "selfdrive/ui/ui.h"
@ -21,24 +17,16 @@ class OnroadAlerts : public QWidget {
Q_OBJECT
public:
OnroadAlerts(QWidget *parent = 0);
OnroadAlerts(QWidget *parent = 0) {};
protected:
void paintEvent(QPaintEvent*) override;
private:
void stopSounds();
void playSound(AudibleAlert alert);
void updateAlert(const QString &t1, const QString &t2, float blink_rate,
const std::string &type, cereal::ControlsState::AlertSize size, AudibleAlert sound);
QColor bg;
float volume = Hardware::MIN_VOLUME;
std::map<AudibleAlert, std::pair<QSoundEffect, int>> sounds;
float blinking_rate = 0;
QString text1, text2;
std::string alert_type;
cereal::ControlsState::AlertSize alert_size;
Alert alert;
void updateAlert(Alert a);
public slots:
void updateState(const UIState &s);

3
selfdrive/ui/soundd 100755
View File

@ -0,0 +1,3 @@
#!/bin/sh
export LD_LIBRARY_PATH="/system/lib64:$LD_LIBRARY_PATH"
exec ./_soundd

100
selfdrive/ui/soundd.cc 100644
View File

@ -0,0 +1,100 @@
#include <sys/resource.h>
#include <map>
#include <QApplication>
#include <QString>
#include <QSoundEffect>
#include "cereal/messaging/messaging.h"
#include "selfdrive/common/util.h"
#include "selfdrive/hardware/hw.h"
#include "selfdrive/ui/ui.h"
// TODO: detect when we can't play sounds
// TODO: detect when we can't display the UI
class Sound : public QObject {
public:
explicit Sound(QObject *parent = 0) {
std::tuple<AudibleAlert, QString, bool> sound_list[] = {
{AudibleAlert::CHIME_DISENGAGE, "../assets/sounds/disengaged.wav", false},
{AudibleAlert::CHIME_ENGAGE, "../assets/sounds/engaged.wav", false},
{AudibleAlert::CHIME_WARNING1, "../assets/sounds/warning_1.wav", false},
{AudibleAlert::CHIME_WARNING2, "../assets/sounds/warning_2.wav", false},
{AudibleAlert::CHIME_WARNING2_REPEAT, "../assets/sounds/warning_2.wav", true},
{AudibleAlert::CHIME_WARNING_REPEAT, "../assets/sounds/warning_repeat.wav", true},
{AudibleAlert::CHIME_ERROR, "../assets/sounds/error.wav", false},
{AudibleAlert::CHIME_PROMPT, "../assets/sounds/error.wav", false}
};
for (auto &[alert, fn, loops] : sound_list) {
sounds[alert].first.setSource(QUrl::fromLocalFile(fn));
sounds[alert].second = loops ? QSoundEffect::Infinite : 0;
}
sm = new SubMaster({"carState", "controlsState"});
QTimer *timer = new QTimer(this);
QObject::connect(timer, &QTimer::timeout, this, &Sound::update);
timer->start();
};
~Sound() {
delete sm;
};
private slots:
void update() {
sm->update(100);
if (sm->updated("carState")) {
// scale volume with speed
volume = util::map_val((*sm)["carState"].getCarState().getVEgo(), 0.f, 20.f,
Hardware::MIN_VOLUME, Hardware::MAX_VOLUME);
}
if (sm->updated("controlsState")) {
const cereal::ControlsState::Reader &cs = (*sm)["controlsState"].getControlsState();
setAlert({QString::fromStdString(cs.getAlertText1()),
QString::fromStdString(cs.getAlertText2()),
QString::fromStdString(cs.getAlertType()),
cs.getAlertSize(), cs.getAlertSound()});
} else if (sm->rcv_frame("controlsState") > 0 && (*sm)["controlsState"].getControlsState().getEnabled() &&
((nanos_since_boot() - sm->rcv_time("controlsState")) / 1e9 > CONTROLS_TIMEOUT)) {
setAlert(CONTROLS_UNRESPONSIVE_ALERT);
}
}
void setAlert(Alert a) {
if (!alert.equal(a)) {
alert = a;
// stop sounds
for (auto &kv : sounds) {
// Only stop repeating sounds
auto &[sound, loops] = kv.second;
if (sound.loopsRemaining() == QSoundEffect::Infinite) {
sound.stop();
}
}
// play sound
if (alert.sound != AudibleAlert::NONE) {
auto &[sound, loops] = sounds[alert.sound];
sound.setLoopCount(loops);
sound.setVolume(volume);
sound.play();
}
}
}
private:
Alert alert;
float volume = Hardware::MIN_VOLUME;
std::map<AudibleAlert, std::pair<QSoundEffect, int>> sounds;
SubMaster *sm;
};
int main(int argc, char **argv) {
setpriority(PRIO_PROCESS, 0, -20);
QApplication a(argc, argv);
Sound sound;
return a.exec();
}

16
selfdrive/test/test_sounds.py → selfdrive/ui/tests/test_sounds.py
View File

@ -4,7 +4,7 @@ import subprocess
from cereal import log, car
import cereal.messaging as messaging
from selfdrive.test.helpers import phone_only, with_processes, set_params_enabled
from selfdrive.test.helpers import phone_only, with_processes
from common.realtime import DT_CTRL
from selfdrive.hardware import HARDWARE
@ -34,10 +34,9 @@ def test_sound_card_init():
@phone_only
@with_processes(['ui', 'camerad'])
@with_processes(['soundd'])
def test_alert_sounds():
set_params_enabled()
pm = messaging.PubMaster(['deviceState', 'controlsState'])
pm = messaging.PubMaster(['controlsState'])
# make sure they're all defined
alert_sounds = {v: k for k, v in car.CarControl.HUDControl.AudibleAlert.schema.enumerants.items()}
@ -45,11 +44,7 @@ def test_alert_sounds():
assert len(diff) == 0, f"not all sounds defined in test: {diff}"
# wait for procs to init
time.sleep(5)
msg = messaging.new_message('deviceState')
msg.deviceState.started = True
pm.send('deviceState', msg)
time.sleep(1)
for sound, expected_writes in SOUNDS.items():
print(f"testing {alert_sounds[sound]}")
@ -57,8 +52,6 @@ def test_alert_sounds():
for _ in range(int(9 / DT_CTRL)):
msg = messaging.new_message('controlsState')
msg.controlsState.enabled = True
msg.controlsState.active = True
msg.controlsState.alertSound = sound
msg.controlsState.alertType = str(sound)
msg.controlsState.alertText1 = "Testing Sounds"
@ -70,4 +63,3 @@ def test_alert_sounds():
tolerance = (expected_writes % 100) * 2
actual_writes = get_total_writes() - start_writes
assert abs(expected_writes - actual_writes) <= tolerance, f"{alert_sounds[sound]}: expected {expected_writes} writes, got {actual_writes}"
#print(f"{alert_sounds[sound]}: expected {expected_writes} writes, got {actual_writes}")

22
selfdrive/ui/ui.h
View File

@ -31,6 +31,8 @@
#define COLOR_YELLOW nvgRGBA(218, 202, 37, 255)
#define COLOR_RED nvgRGBA(201, 34, 49, 255)
typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert;
// TODO: this is also hardcoded in common/transformations/camera.py
// TODO: choose based on frame input size
const float y_offset = Hardware::TICI() ? 150.0 : 0.0;
@ -47,6 +49,26 @@ typedef struct Rect {
}
} Rect;
typedef struct Alert {
QString text1;
QString text2;
QString type;
cereal::ControlsState::AlertSize size;
AudibleAlert sound;
bool equal(Alert a2) {
return text1 == a2.text1 && text2 == a2.text2 && type == a2.type;
}
} Alert;
const Alert CONTROLS_WAITING_ALERT = {"openpilot Unavailable", "Waiting for controls to start",
"controlsWaiting", cereal::ControlsState::AlertSize::MID,
AudibleAlert::NONE};
const Alert CONTROLS_UNRESPONSIVE_ALERT = {"TAKE CONTROL IMMEDIATELY", "Controls Unresponsive",
"controlsUnresponsive", cereal::ControlsState::AlertSize::FULL,
AudibleAlert::CHIME_WARNING_REPEAT};
const int CONTROLS_TIMEOUT = 5;
const int bdr_s = 30;
const int header_h = 420;
const int footer_h = 280;