From 12948e661a6b6424739082ee614d81b4549c2753 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 16 Jul 2021 17:30:00 -0700 Subject: [PATCH] soundd (#21619) * refactor alerts * doesn't go here anymore * soudd * handle controls unresponsive * same check * fix path * update sound test * nice * fix c2 * add script * update tests Co-authored-by: Comma Device --- .gitignore | 1 + Jenkinsfile | 2 +- selfdrive/manager/process_config.py | 1 + selfdrive/test/test_onroad.py | 5 +- selfdrive/ui/SConscript | 3 + selfdrive/ui/main.cc | 4 + selfdrive/ui/qt/onroad.cc | 102 +++++--------------- selfdrive/ui/qt/onroad.h | 20 +--- selfdrive/ui/soundd | 3 + selfdrive/ui/soundd.cc | 100 +++++++++++++++++++ selfdrive/{test => ui/tests}/test_sounds.py | 16 +-- selfdrive/ui/ui.h | 22 +++++ 12 files changed, 169 insertions(+), 110 deletions(-) create mode 100755 selfdrive/ui/soundd create mode 100644 selfdrive/ui/soundd.cc rename selfdrive/{test => ui/tests}/test_sounds.py (81%) diff --git a/.gitignore b/.gitignore index f4932ed4f..14ba0bd05 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ selfdrive/logcatd/logcatd selfdrive/mapd/default_speeds_by_region.json selfdrive/proclogd/proclogd selfdrive/ui/_ui +selfdrive/ui/_soundd selfdrive/test/longitudinal_maneuvers/out selfdrive/visiond/visiond selfdrive/loggerd/loggerd diff --git a/Jenkinsfile b/Jenkinsfile index dbcc11066..9544a4f7e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -152,7 +152,7 @@ pipeline { phone_steps("eon", [ ["build", "cd selfdrive/manager && ./build.py"], ["test athena", "nosetests -s selfdrive/athena/tests/test_athenad_old.py"], - ["test sounds", "nosetests -s selfdrive/test/test_sounds.py"], + ["test sounds", "nosetests -s selfdrive/ui/tests/test_sounds.py"], ["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"], ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"], ["test encoder", "python selfdrive/loggerd/tests/test_encoder.py"], diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index 9ed6d9f87..db7906d8a 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -18,6 +18,7 @@ procs = [ NativeProcess("sensord", "selfdrive/sensord", ["./sensord"], enabled=not PC, persistent=EON, sigkill=EON), NativeProcess("ubloxd", "selfdrive/locationd", ["./ubloxd"], enabled=(not PC or WEBCAM)), NativeProcess("ui", "selfdrive/ui", ["./ui"], persistent=True, watchdog_max_dt=(10 if TICI else None)), + NativeProcess("soundd", "selfdrive/ui", ["./soundd"]), NativeProcess("locationd", "selfdrive/locationd", ["./locationd"]), PythonProcess("calibrationd", "selfdrive.locationd.calibrationd"), PythonProcess("controlsd", "selfdrive.controls.controlsd"), diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 7bd228fc7..e63ef5ebf 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -34,6 +34,7 @@ PROCS = { "./_dmonitoringmodeld": 2.67, "selfdrive.thermald.thermald": 2.41, "selfdrive.locationd.calibrationd": 2.0, + "./soundd": 2.0, "selfdrive.monitoring.dmonitoringd": 1.90, "./proclogd": 1.54, "selfdrive.logmessaged": 0.2, @@ -48,6 +49,7 @@ if TICI: "./loggerd": 60.0, "selfdrive.controls.controlsd": 26.0, "./camerad": 25.0, + "./_ui": 21.0, "selfdrive.controls.plannerd": 12.0, "selfdrive.locationd.paramsd": 5.0, "./_dmonitoringmodeld": 10.0, @@ -74,9 +76,6 @@ def check_cpu_usage(first_proc, last_proc): cpu_time = cputime_total(last) - cputime_total(first) cpu_usage = cpu_time / dt * 100. if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0): - # TODO: fix high CPU when playing sounds constantly in UI - if proc_name == "./_ui" and cpu_usage < 50.: - continue result += f"Warning {proc_name} using more CPU than normal\n" r = False elif cpu_usage < min(normal_cpu_usage * 0.65, max(normal_cpu_usage - 1.0, 0.0)): diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index ffa34c3ee..548be88a7 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -39,6 +39,9 @@ elif maps: widgets = qt_env.Library("qt_widgets", widgets_src, LIBS=base_libs) qt_libs = [widgets] + base_libs +# build soundd +qt_env.Program("_soundd", "soundd.cc", LIBS=base_libs) + # spinner and text window qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs) qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=qt_libs) diff --git a/selfdrive/ui/main.cc b/selfdrive/ui/main.cc index 63bbffa7a..3d2411b9c 100644 --- a/selfdrive/ui/main.cc +++ b/selfdrive/ui/main.cc @@ -1,3 +1,5 @@ +#include + #include #include @@ -7,6 +9,8 @@ #include "selfdrive/ui/qt/window.h" int main(int argc, char *argv[]) { + setpriority(PRIO_PROCESS, 0, -20); + qInstallMessageHandler(swagLogMessageHandler); initApp(); diff --git a/selfdrive/ui/qt/onroad.cc b/selfdrive/ui/qt/onroad.cc index 296f063e8..10fc19852 100644 --- a/selfdrive/ui/qt/onroad.cc +++ b/selfdrive/ui/qt/onroad.cc @@ -67,45 +67,23 @@ void OnroadWindow::offroadTransition(bool offroad) { // ***** onroad widgets ***** -OnroadAlerts::OnroadAlerts(QWidget *parent) : QWidget(parent) { - std::tuple sound_list[] = { - {AudibleAlert::CHIME_DISENGAGE, "../assets/sounds/disengaged.wav", false}, - {AudibleAlert::CHIME_ENGAGE, "../assets/sounds/engaged.wav", false}, - {AudibleAlert::CHIME_WARNING1, "../assets/sounds/warning_1.wav", false}, - {AudibleAlert::CHIME_WARNING2, "../assets/sounds/warning_2.wav", false}, - {AudibleAlert::CHIME_WARNING2_REPEAT, "../assets/sounds/warning_2.wav", true}, - {AudibleAlert::CHIME_WARNING_REPEAT, "../assets/sounds/warning_repeat.wav", true}, - {AudibleAlert::CHIME_ERROR, "../assets/sounds/error.wav", false}, - {AudibleAlert::CHIME_PROMPT, "../assets/sounds/error.wav", false}}; - - for (auto &[alert, fn, loops] : sound_list) { - sounds[alert].first.setSource(QUrl::fromLocalFile(fn)); - sounds[alert].second = loops ? QSoundEffect::Infinite : 0; - } -} - void OnroadAlerts::updateState(const UIState &s) { SubMaster &sm = *(s.sm); - if (sm.updated("carState")) { - // scale volume with speed - volume = util::map_val(sm["carState"].getCarState().getVEgo(), 0.f, 20.f, - Hardware::MIN_VOLUME, Hardware::MAX_VOLUME); - } if (sm["deviceState"].getDeviceState().getStarted()) { if (sm.updated("controlsState")) { const cereal::ControlsState::Reader &cs = sm["controlsState"].getControlsState(); - updateAlert(QString::fromStdString(cs.getAlertText1()), QString::fromStdString(cs.getAlertText2()), - cs.getAlertBlinkingRate(), cs.getAlertType(), cs.getAlertSize(), cs.getAlertSound()); - } else if ((sm.frame - s.scene.started_frame) > 10 * UI_FREQ) { + updateAlert({QString::fromStdString(cs.getAlertText1()), + QString::fromStdString(cs.getAlertText2()), + QString::fromStdString(cs.getAlertType()), + cs.getAlertSize(), cs.getAlertSound()}); + } else if ((sm.frame - s.scene.started_frame) > 5 * UI_FREQ) { // Handle controls timeout if (sm.rcv_frame("controlsState") < s.scene.started_frame) { // car is started, but controlsState hasn't been seen at all - updateAlert("openpilot Unavailable", "Waiting for controls to start", 0, - "controlsWaiting", cereal::ControlsState::AlertSize::MID, AudibleAlert::NONE); - } else if ((sm.frame - sm.rcv_frame("controlsState")) > 5 * UI_FREQ) { + updateAlert(CONTROLS_WAITING_ALERT); + } else if ((nanos_since_boot() - sm.rcv_time("controlsState")) / 1e9 > CONTROLS_TIMEOUT) { // car is started, but controls is lagging or died - updateAlert("TAKE CONTROL IMMEDIATELY", "Controls Unresponsive", 0, - "controlsUnresponsive", cereal::ControlsState::AlertSize::FULL, AudibleAlert::CHIME_WARNING_REPEAT); + updateAlert(CONTROLS_UNRESPONSIVE_ALERT); // TODO: clean this up once Qt handles the border QUIState::ui_state.status = STATUS_ALERT; @@ -113,54 +91,22 @@ void OnroadAlerts::updateState(const UIState &s) { } } - // TODO: add blinking back if performant - //float alpha = 0.375 * cos((millis_since_boot() / 1000) * 2 * M_PI * blinking_rate) + 0.625; bg = bg_colors[s.status]; } void OnroadAlerts::offroadTransition(bool offroad) { - updateAlert("", "", 0, "", cereal::ControlsState::AlertSize::NONE, AudibleAlert::NONE); + updateAlert({}); } -void OnroadAlerts::updateAlert(const QString &t1, const QString &t2, float blink_rate, - const std::string &type, cereal::ControlsState::AlertSize size, AudibleAlert sound) { - if (alert_type.compare(type) == 0 && text1.compare(t1) == 0 && text2.compare(t2) == 0) { - return; - } - - stopSounds(); - if (sound != AudibleAlert::NONE) { - playSound(sound); - } - - text1 = t1; - text2 = t2; - alert_type = type; - alert_size = size; - blinking_rate = blink_rate; - - update(); -} - -void OnroadAlerts::playSound(AudibleAlert alert) { - auto &[sound, loops] = sounds[alert]; - sound.setLoopCount(loops); - sound.setVolume(volume); - sound.play(); -} - -void OnroadAlerts::stopSounds() { - for (auto &kv : sounds) { - // Only stop repeating sounds - auto &[sound, loops] = kv.second; - if (sound.loopsRemaining() == QSoundEffect::Infinite) { - sound.stop(); - } +void OnroadAlerts::updateAlert(Alert a) { + if (!alert.equal(a)) { + alert = a; + update(); } } void OnroadAlerts::paintEvent(QPaintEvent *event) { - if (alert_size == cereal::ControlsState::AlertSize::NONE) { + if (alert.size == cereal::ControlsState::AlertSize::NONE) { return; } static std::map alert_sizes = { @@ -168,7 +114,7 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) { {cereal::ControlsState::AlertSize::MID, 420}, {cereal::ControlsState::AlertSize::FULL, height()}, }; - int h = alert_sizes[alert_size]; + int h = alert_sizes[alert.size]; QRect r = QRect(0, height() - h, width(), h); QPainter p(this); @@ -196,20 +142,20 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) { const QPoint c = r.center(); p.setPen(QColor(0xff, 0xff, 0xff)); p.setRenderHint(QPainter::TextAntialiasing); - if (alert_size == cereal::ControlsState::AlertSize::SMALL) { + if (alert.size == cereal::ControlsState::AlertSize::SMALL) { configFont(p, "Open Sans", 74, "SemiBold"); - p.drawText(r, Qt::AlignCenter, text1); - } else if (alert_size == cereal::ControlsState::AlertSize::MID) { + p.drawText(r, Qt::AlignCenter, alert.text1); + } else if (alert.size == cereal::ControlsState::AlertSize::MID) { configFont(p, "Open Sans", 88, "Bold"); - p.drawText(QRect(0, c.y() - 125, width(), 150), Qt::AlignHCenter | Qt::AlignTop, text1); + p.drawText(QRect(0, c.y() - 125, width(), 150), Qt::AlignHCenter | Qt::AlignTop, alert.text1); configFont(p, "Open Sans", 66, "Regular"); - p.drawText(QRect(0, c.y() + 21, width(), 90), Qt::AlignHCenter, text2); - } else if (alert_size == cereal::ControlsState::AlertSize::FULL) { - bool l = text1.length() > 15; + p.drawText(QRect(0, c.y() + 21, width(), 90), Qt::AlignHCenter, alert.text2); + } else if (alert.size == cereal::ControlsState::AlertSize::FULL) { + bool l = alert.text1.length() > 15; configFont(p, "Open Sans", l ? 132 : 177, "Bold"); - p.drawText(QRect(0, r.y() + (l ? 240 : 270), width(), 600), Qt::AlignHCenter | Qt::TextWordWrap, text1); + p.drawText(QRect(0, r.y() + (l ? 240 : 270), width(), 600), Qt::AlignHCenter | Qt::TextWordWrap, alert.text1); configFont(p, "Open Sans", 88, "Regular"); - p.drawText(QRect(0, r.height() - (l ? 361 : 420), width(), 300), Qt::AlignHCenter | Qt::TextWordWrap, text2); + p.drawText(QRect(0, r.height() - (l ? 361 : 420), width(), 300), Qt::AlignHCenter | Qt::TextWordWrap, alert.text2); } } diff --git a/selfdrive/ui/qt/onroad.h b/selfdrive/ui/qt/onroad.h index ae2cfda8f..2120cdc78 100644 --- a/selfdrive/ui/qt/onroad.h +++ b/selfdrive/ui/qt/onroad.h @@ -1,15 +1,11 @@ #pragma once -#include - #include #include -#include #include #include #include "cereal/gen/cpp/log.capnp.h" -#include "selfdrive/hardware/hw.h" #include "selfdrive/ui/qt/qt_window.h" #include "selfdrive/ui/ui.h" @@ -21,24 +17,16 @@ class OnroadAlerts : public QWidget { Q_OBJECT public: - OnroadAlerts(QWidget *parent = 0); + OnroadAlerts(QWidget *parent = 0) {}; protected: void paintEvent(QPaintEvent*) override; private: - void stopSounds(); - void playSound(AudibleAlert alert); - void updateAlert(const QString &t1, const QString &t2, float blink_rate, - const std::string &type, cereal::ControlsState::AlertSize size, AudibleAlert sound); - QColor bg; - float volume = Hardware::MIN_VOLUME; - std::map> sounds; - float blinking_rate = 0; - QString text1, text2; - std::string alert_type; - cereal::ControlsState::AlertSize alert_size; + Alert alert; + + void updateAlert(Alert a); public slots: void updateState(const UIState &s); diff --git a/selfdrive/ui/soundd b/selfdrive/ui/soundd new file mode 100755 index 000000000..e658062b7 --- /dev/null +++ b/selfdrive/ui/soundd @@ -0,0 +1,3 @@ +#!/bin/sh +export LD_LIBRARY_PATH="/system/lib64:$LD_LIBRARY_PATH" +exec ./_soundd diff --git a/selfdrive/ui/soundd.cc b/selfdrive/ui/soundd.cc new file mode 100644 index 000000000..33fe57b9a --- /dev/null +++ b/selfdrive/ui/soundd.cc @@ -0,0 +1,100 @@ +#include + +#include + +#include +#include +#include + +#include "cereal/messaging/messaging.h" +#include "selfdrive/common/util.h" +#include "selfdrive/hardware/hw.h" +#include "selfdrive/ui/ui.h" + +// TODO: detect when we can't play sounds +// TODO: detect when we can't display the UI + +class Sound : public QObject { +public: + explicit Sound(QObject *parent = 0) { + std::tuple sound_list[] = { + {AudibleAlert::CHIME_DISENGAGE, "../assets/sounds/disengaged.wav", false}, + {AudibleAlert::CHIME_ENGAGE, "../assets/sounds/engaged.wav", false}, + {AudibleAlert::CHIME_WARNING1, "../assets/sounds/warning_1.wav", false}, + {AudibleAlert::CHIME_WARNING2, "../assets/sounds/warning_2.wav", false}, + {AudibleAlert::CHIME_WARNING2_REPEAT, "../assets/sounds/warning_2.wav", true}, + {AudibleAlert::CHIME_WARNING_REPEAT, "../assets/sounds/warning_repeat.wav", true}, + {AudibleAlert::CHIME_ERROR, "../assets/sounds/error.wav", false}, + {AudibleAlert::CHIME_PROMPT, "../assets/sounds/error.wav", false} + }; + for (auto &[alert, fn, loops] : sound_list) { + sounds[alert].first.setSource(QUrl::fromLocalFile(fn)); + sounds[alert].second = loops ? QSoundEffect::Infinite : 0; + } + + sm = new SubMaster({"carState", "controlsState"}); + + QTimer *timer = new QTimer(this); + QObject::connect(timer, &QTimer::timeout, this, &Sound::update); + timer->start(); + }; + ~Sound() { + delete sm; + }; + +private slots: + void update() { + sm->update(100); + if (sm->updated("carState")) { + // scale volume with speed + volume = util::map_val((*sm)["carState"].getCarState().getVEgo(), 0.f, 20.f, + Hardware::MIN_VOLUME, Hardware::MAX_VOLUME); + } + if (sm->updated("controlsState")) { + const cereal::ControlsState::Reader &cs = (*sm)["controlsState"].getControlsState(); + setAlert({QString::fromStdString(cs.getAlertText1()), + QString::fromStdString(cs.getAlertText2()), + QString::fromStdString(cs.getAlertType()), + cs.getAlertSize(), cs.getAlertSound()}); + } else if (sm->rcv_frame("controlsState") > 0 && (*sm)["controlsState"].getControlsState().getEnabled() && + ((nanos_since_boot() - sm->rcv_time("controlsState")) / 1e9 > CONTROLS_TIMEOUT)) { + setAlert(CONTROLS_UNRESPONSIVE_ALERT); + } + } + + void setAlert(Alert a) { + if (!alert.equal(a)) { + alert = a; + // stop sounds + for (auto &kv : sounds) { + // Only stop repeating sounds + auto &[sound, loops] = kv.second; + if (sound.loopsRemaining() == QSoundEffect::Infinite) { + sound.stop(); + } + } + + // play sound + if (alert.sound != AudibleAlert::NONE) { + auto &[sound, loops] = sounds[alert.sound]; + sound.setLoopCount(loops); + sound.setVolume(volume); + sound.play(); + } + } + } + +private: + Alert alert; + float volume = Hardware::MIN_VOLUME; + std::map> sounds; + SubMaster *sm; +}; + +int main(int argc, char **argv) { + setpriority(PRIO_PROCESS, 0, -20); + + QApplication a(argc, argv); + Sound sound; + return a.exec(); +} diff --git a/selfdrive/test/test_sounds.py b/selfdrive/ui/tests/test_sounds.py similarity index 81% rename from selfdrive/test/test_sounds.py rename to selfdrive/ui/tests/test_sounds.py index 20c0d2be9..1f81370dc 100755 --- a/selfdrive/test/test_sounds.py +++ b/selfdrive/ui/tests/test_sounds.py @@ -4,7 +4,7 @@ import subprocess from cereal import log, car import cereal.messaging as messaging -from selfdrive.test.helpers import phone_only, with_processes, set_params_enabled +from selfdrive.test.helpers import phone_only, with_processes from common.realtime import DT_CTRL from selfdrive.hardware import HARDWARE @@ -34,10 +34,9 @@ def test_sound_card_init(): @phone_only -@with_processes(['ui', 'camerad']) +@with_processes(['soundd']) def test_alert_sounds(): - set_params_enabled() - pm = messaging.PubMaster(['deviceState', 'controlsState']) + pm = messaging.PubMaster(['controlsState']) # make sure they're all defined alert_sounds = {v: k for k, v in car.CarControl.HUDControl.AudibleAlert.schema.enumerants.items()} @@ -45,11 +44,7 @@ def test_alert_sounds(): assert len(diff) == 0, f"not all sounds defined in test: {diff}" # wait for procs to init - time.sleep(5) - - msg = messaging.new_message('deviceState') - msg.deviceState.started = True - pm.send('deviceState', msg) + time.sleep(1) for sound, expected_writes in SOUNDS.items(): print(f"testing {alert_sounds[sound]}") @@ -57,8 +52,6 @@ def test_alert_sounds(): for _ in range(int(9 / DT_CTRL)): msg = messaging.new_message('controlsState') - msg.controlsState.enabled = True - msg.controlsState.active = True msg.controlsState.alertSound = sound msg.controlsState.alertType = str(sound) msg.controlsState.alertText1 = "Testing Sounds" @@ -70,4 +63,3 @@ def test_alert_sounds(): tolerance = (expected_writes % 100) * 2 actual_writes = get_total_writes() - start_writes assert abs(expected_writes - actual_writes) <= tolerance, f"{alert_sounds[sound]}: expected {expected_writes} writes, got {actual_writes}" - #print(f"{alert_sounds[sound]}: expected {expected_writes} writes, got {actual_writes}") diff --git a/selfdrive/ui/ui.h b/selfdrive/ui/ui.h index db782c0a8..8b43449ed 100644 --- a/selfdrive/ui/ui.h +++ b/selfdrive/ui/ui.h @@ -31,6 +31,8 @@ #define COLOR_YELLOW nvgRGBA(218, 202, 37, 255) #define COLOR_RED nvgRGBA(201, 34, 49, 255) +typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert; + // TODO: this is also hardcoded in common/transformations/camera.py // TODO: choose based on frame input size const float y_offset = Hardware::TICI() ? 150.0 : 0.0; @@ -47,6 +49,26 @@ typedef struct Rect { } } Rect; +typedef struct Alert { + QString text1; + QString text2; + QString type; + cereal::ControlsState::AlertSize size; + AudibleAlert sound; + bool equal(Alert a2) { + return text1 == a2.text1 && text2 == a2.text2 && type == a2.type; + } +} Alert; + +const Alert CONTROLS_WAITING_ALERT = {"openpilot Unavailable", "Waiting for controls to start", + "controlsWaiting", cereal::ControlsState::AlertSize::MID, + AudibleAlert::NONE}; + +const Alert CONTROLS_UNRESPONSIVE_ALERT = {"TAKE CONTROL IMMEDIATELY", "Controls Unresponsive", + "controlsUnresponsive", cereal::ControlsState::AlertSize::FULL, + AudibleAlert::CHIME_WARNING_REPEAT}; +const int CONTROLS_TIMEOUT = 5; + const int bdr_s = 30; const int header_h = 420; const int footer_h = 280;