spacecruft
/
nopenpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

UI: move rest of NVG to Qt (#23133) 

* move ui_draw_world to qt

* draw world if world_objects_visible is true

* cleanup

* cleanup includes

* continue

* cleanup

* cleanup

* enable  multisampling on all devices
pull/23147/head
Dean Lee 2021-12-07 06:02:39 +08:00 committed by GitHub
parent 5e20a46db6
commit 20f571db3c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 126 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
selfdrive/ui/SConscript
View File

@ -56,10 +56,9 @@ qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs)
qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=qt_libs)
# build main UI
qt_src = ["main.cc", "ui.cc", "paint.cc", "qt/sidebar.cc", "qt/onroad.cc",
qt_src = ["main.cc", "ui.cc", "qt/sidebar.cc", "qt/onroad.cc",
"qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc",
"qt/offroad/onboarding.cc", "qt/offroad/driverview.cc",
"#third_party/nanovg/nanovg.c"]
"qt/offroad/onboarding.cc", "qt/offroad/driverview.cc"]
qt_env.Program("_ui", qt_src + [asset_obj], LIBS=qt_libs)

175
selfdrive/ui/paint.cc
View File

@ -1,175 +0,0 @@
#include "selfdrive/ui/paint.h"
#include <cassert>
#ifdef __APPLE__
#include <OpenGL/gl3.h>
#define NANOVG_GL3_IMPLEMENTATION
#define nvgCreate nvgCreateGL3
#else
#include <GLES3/gl3.h>
#define NANOVG_GLES3_IMPLEMENTATION
#define nvgCreate nvgCreateGLES3
#endif
#define NANOVG_GLES3_IMPLEMENTATION
#include <nanovg_gl.h>
#include <nanovg_gl_utils.h>
#include "selfdrive/hardware/hw.h"
static void draw_chevron(UIState *s, float x, float y, float sz, NVGcolor fillColor, NVGcolor glowColor) {
// glow
float g_xo = sz/5;
float g_yo = sz/10;
nvgBeginPath(s->vg);
nvgMoveTo(s->vg, x+(sz*1.35)+g_xo, y+sz+g_yo);
nvgLineTo(s->vg, x, y-g_xo);
nvgLineTo(s->vg, x-(sz*1.35)-g_xo, y+sz+g_yo);
nvgClosePath(s->vg);
nvgFillColor(s->vg, glowColor);
nvgFill(s->vg);
// chevron
nvgBeginPath(s->vg);
nvgMoveTo(s->vg, x+(sz*1.25), y+sz);
nvgLineTo(s->vg, x, y);
nvgLineTo(s->vg, x-(sz*1.25), y+sz);
nvgClosePath(s->vg);
nvgFillColor(s->vg, fillColor);
nvgFill(s->vg);
}
static void draw_lead(UIState *s, const cereal::RadarState::LeadData::Reader &lead_data, const vertex_data &vd) {
// Draw lead car indicator
auto [x, y] = vd;
float fillAlpha = 0;
float speedBuff = 10.;
float leadBuff = 40.;
float d_rel = lead_data.getDRel();
float v_rel = lead_data.getVRel();
if (d_rel < leadBuff) {
fillAlpha = 255*(1.0-(d_rel/leadBuff));
if (v_rel < 0) {
fillAlpha += 255*(-1*(v_rel/speedBuff));
}
fillAlpha = (int)(fmin(fillAlpha, 255));
}
float sz = std::clamp((25 * 30) / (d_rel / 3 + 30), 15.0f, 30.0f) * 2.35;
x = std::clamp(x, 0.f, s->fb_w - sz / 2);
y = std::fmin(s->fb_h - sz * .6, y);
draw_chevron(s, x, y, sz, nvgRGBA(201, 34, 49, fillAlpha), COLOR_YELLOW);
}
static void ui_draw_line(UIState *s, const line_vertices_data &vd, NVGcolor *color, NVGpaint *paint) {
if (vd.cnt == 0) return;
const vertex_data *v = &vd.v[0];
nvgBeginPath(s->vg);
nvgMoveTo(s->vg, v[0].x, v[0].y);
for (int i = 1; i < vd.cnt; i++) {
nvgLineTo(s->vg, v[i].x, v[i].y);
}
nvgClosePath(s->vg);
if (color) {
nvgFillColor(s->vg, *color);
} else if (paint) {
nvgFillPaint(s->vg, *paint);
}
nvgFill(s->vg);
}
static void ui_draw_vision_lane_lines(UIState *s) {
const UIScene &scene = s->scene;
NVGpaint track_bg;
if (!scene.end_to_end) {
// paint lanelines
for (int i = 0; i < std::size(scene.lane_line_vertices); i++) {
NVGcolor color = nvgRGBAf(1.0, 1.0, 1.0, scene.lane_line_probs[i]);
ui_draw_line(s, scene.lane_line_vertices[i], &color, nullptr);
}
// paint road edges
for (int i = 0; i < std::size(scene.road_edge_vertices); i++) {
NVGcolor color = nvgRGBAf(1.0, 0.0, 0.0, std::clamp<float>(1.0 - scene.road_edge_stds[i], 0.0, 1.0));
ui_draw_line(s, scene.road_edge_vertices[i], &color, nullptr);
}
track_bg = nvgLinearGradient(s->vg, s->fb_w, s->fb_h, s->fb_w, s->fb_h * .4,
COLOR_WHITE, COLOR_WHITE_ALPHA(0));
} else {
track_bg = nvgLinearGradient(s->vg, s->fb_w, s->fb_h, s->fb_w, s->fb_h * .4,
COLOR_RED, COLOR_RED_ALPHA(0));
}
// paint path
ui_draw_line(s, scene.track_vertices, nullptr, &track_bg);
}
// Draw all world space objects.
static void ui_draw_world(UIState *s) {
nvgScissor(s->vg, 0, 0, s->fb_w, s->fb_h);
// Draw lane edges and vision/mpc tracks
ui_draw_vision_lane_lines(s);
// Draw lead indicators if openpilot is handling longitudinal
if (s->scene.longitudinal_control) {
auto lead_one = (*s->sm)["radarState"].getRadarState().getLeadOne();
auto lead_two = (*s->sm)["radarState"].getRadarState().getLeadTwo();
if (lead_one.getStatus()) {
draw_lead(s, lead_one, s->scene.lead_vertices[0]);
}
if (lead_two.getStatus() && (std::abs(lead_one.getDRel() - lead_two.getDRel()) > 3.0)) {
draw_lead(s, lead_two, s->scene.lead_vertices[1]);
}
}
nvgResetScissor(s->vg);
}
static void ui_draw_vision(UIState *s) {
const UIScene *scene = &s->scene;
// Draw augmented elements
if (scene->world_objects_visible) {
ui_draw_world(s);
}
}
void ui_draw(UIState *s, int w, int h) {
// Update intrinsics matrix after possible wide camera toggle change
if (s->fb_w != w || s->fb_h != h) {
ui_resize(s, w, h);
}
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
nvgBeginFrame(s->vg, s->fb_w, s->fb_h, 1.0f);
ui_draw_vision(s);
nvgEndFrame(s->vg);
glDisable(GL_BLEND);
}
void ui_nvg_init(UIState *s) {
// on EON, we enable MSAA
s->vg = Hardware::EON() ? nvgCreate(0) : nvgCreate(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG);
assert(s->vg);
}
void ui_resize(UIState *s, int width, int height) {
s->fb_w = width;
s->fb_h = height;
auto intrinsic_matrix = s->wide_camera ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
float zoom = ZOOM / intrinsic_matrix.v[0];
if (s->wide_camera) {
zoom *= 0.5;
}
// Apply transformation such that video pixel coordinates match video
// 1) Put (0, 0) in the middle of the video
// 2) Apply same scaling as video
// 3) Put (0, 0) in top left corner of video
s->car_space_transform.reset();
s->car_space_transform.translate(width / 2, height / 2 + y_offset)
.scale(zoom, zoom)
.translate(-intrinsic_matrix.v[2], -intrinsic_matrix.v[5]);
}

8
selfdrive/ui/paint.h
View File

@ -1,8 +0,0 @@
#pragma once
#include "selfdrive/ui/ui.h"
void ui_draw(UIState *s, int w, int h);
void ui_nvg_init(UIState *s);
void ui_resize(UIState *s, int width, int height);
void ui_update_params(UIState *s);

97
selfdrive/ui/qt/onroad.cc
View File

@ -3,7 +3,6 @@
#include <QDebug>
#include "selfdrive/common/timing.h"
#include "selfdrive/ui/paint.h"
#include "selfdrive/ui/qt/util.h"
#ifdef ENABLE_MAPS
#include "selfdrive/ui/qt/maps/map.h"
@ -266,14 +265,105 @@ void NvgWindow::initializeGL() {
qInfo() << "OpenGL renderer:" << QString((const char*)glGetString(GL_RENDERER));
qInfo() << "OpenGL language version:" << QString((const char*)glGetString(GL_SHADING_LANGUAGE_VERSION));
ui_nvg_init(&QUIState::ui_state);
prev_draw_t = millis_since_boot();
setBackgroundColor(bg_colors[STATUS_DISENGAGED]);
}
void NvgWindow::updateFrameMat(int w, int h) {
CameraViewWidget::updateFrameMat(w, h);
UIState *s = &QUIState::ui_state;
s->fb_w = w;
s->fb_h = h;
auto intrinsic_matrix = s->wide_camera ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
float zoom = ZOOM / intrinsic_matrix.v[0];
if (s->wide_camera) {
zoom *= 0.5;
}
// Apply transformation such that video pixel coordinates match video
// 1) Put (0, 0) in the middle of the video
// 2) Apply same scaling as video
// 3) Put (0, 0) in top left corner of video
s->car_space_transform.reset();
s->car_space_transform.translate(w / 2, h / 2 + y_offset)
.scale(zoom, zoom)
.translate(-intrinsic_matrix.v[2], -intrinsic_matrix.v[5]);
}
void NvgWindow::drawLaneLines(QPainter &painter, const UIScene &scene) {
if (!scene.end_to_end) {
// lanelines
for (int i = 0; i < std::size(scene.lane_line_vertices); ++i) {
painter.setBrush(QColor::fromRgbF(1.0, 1.0, 1.0, scene.lane_line_probs[i]));
painter.drawPolygon(scene.lane_line_vertices[i].v, scene.lane_line_vertices[i].cnt);
}
// road edges
for (int i = 0; i < std::size(scene.road_edge_vertices); ++i) {
painter.setBrush(QColor::fromRgbF(1.0, 0, 0, std::clamp<float>(1.0 - scene.road_edge_stds[i], 0.0, 1.0)));
painter.drawPolygon(scene.road_edge_vertices[i].v, scene.road_edge_vertices[i].cnt);
}
}
// paint path
QLinearGradient bg(0, height(), 0, height() / 4);
bg.setColorAt(0, scene.end_to_end ? redColor() : QColor(255, 255, 255));
bg.setColorAt(1, scene.end_to_end ? redColor(0) : QColor(255, 255, 255, 0));
painter.setBrush(bg);
painter.drawPolygon(scene.track_vertices.v, scene.track_vertices.cnt);
}
void NvgWindow::drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd) {
const float speedBuff = 10.;
const float leadBuff = 40.;
const float d_rel = lead_data.getX()[0];
const float v_rel = lead_data.getV()[0];
float fillAlpha = 0;
if (d_rel < leadBuff) {
fillAlpha = 255 * (1.0 - (d_rel / leadBuff));
if (v_rel < 0) {
fillAlpha += 255 * (-1 * (v_rel / speedBuff));
}
fillAlpha = (int)(fmin(fillAlpha, 255));
}
float sz = std::clamp((25 * 30) / (d_rel / 3 + 30), 15.0f, 30.0f) * 2.35;
float x = std::clamp((float)vd.x(), 0.f, width() - sz / 2);
float y = std::fmin(height() - sz * .6, (float)vd.y());
float g_xo = sz / 5;
float g_yo = sz / 10;
QPointF glow[] = {{x + (sz * 1.35) + g_xo, y + sz + g_yo}, {x, y - g_xo}, {x - (sz * 1.35) - g_xo, y + sz + g_yo}};
painter.setBrush(QColor(218, 202, 37, 255));
painter.drawPolygon(glow, std::size(glow));
// chevron
QPointF chevron[] = {{x + (sz * 1.25), y + sz}, {x, y}, {x - (sz * 1.25), y + sz}};
painter.setBrush(redColor(fillAlpha));
painter.drawPolygon(chevron, std::size(chevron));
}
void NvgWindow::paintGL() {
CameraViewWidget::paintGL();
ui_draw(&QUIState::ui_state, width(), height());
UIState *s = &QUIState::ui_state;
if (s->scene.world_objects_visible) {
QPainter painter(this);
painter.setRenderHint(QPainter::Antialiasing);
painter.setPen(Qt::NoPen);
drawLaneLines(painter, s->scene);
if (s->scene.longitudinal_control) {
auto leads = (*s->sm)["modelV2"].getModelV2().getLeadsV3();
if (leads[0].getProb() > .5) {
drawLead(painter, leads[0], s->scene.lead_vertices[0]);
}
if (leads[1].getProb() > .5 && (std::abs(leads[1].getX()[0] - leads[0].getX()[0]) > 3.0)) {
drawLead(painter, leads[1], s->scene.lead_vertices[1]);
}
}
}
double cur_draw_t = millis_since_boot();
double dt = cur_draw_t - prev_draw_t;
@ -286,6 +376,7 @@ void NvgWindow::paintGL() {
void NvgWindow::showEvent(QShowEvent *event) {
CameraViewWidget::showEvent(event);
ui_update_params(&QUIState::ui_state);
prev_draw_t = millis_since_boot();
}

4
selfdrive/ui/qt/onroad.h
View File

@ -72,6 +72,10 @@ protected:
void paintGL() override;
void initializeGL() override;
void showEvent(QShowEvent *event) override;
void updateFrameMat(int w, int h) override;
void drawLaneLines(QPainter &painter, const UIScene &scene);
void drawLead(QPainter &painter, const cereal::ModelDataV2::LeadDataV3::Reader &lead_data, const QPointF &vd);
inline QColor redColor(int alpha = 255) { return QColor(201, 34, 49, alpha); }
double prev_draw_t = 0;
};

1
selfdrive/ui/qt/util.cc
View File

@ -85,6 +85,7 @@ void setQtSurfaceFormat() {
#else
fmt.setRenderableType(QSurfaceFormat::OpenGLES);
#endif
fmt.setSamples(16);
QSurfaceFormat::setDefaultFormat(fmt);
}

10
selfdrive/ui/qt/widgets/cameraview.cc
View File

@ -1,12 +1,18 @@
#include "selfdrive/ui/qt/widgets/cameraview.h"
#ifdef __APPLE__
#include <OpenGL/gl3.h>
#else
#include <GLES3/gl3.h>
#endif
#include <QOpenGLBuffer>
#include <QOffscreenSurface>
namespace {
const char frame_vertex_shader[] =
#ifdef NANOVG_GL3_IMPLEMENTATION
#ifdef __APPLE__
"#version 150 core\n"
#else
"#version 300 es\n"
@ -21,7 +27,7 @@ const char frame_vertex_shader[] =
"}\n";
const char frame_fragment_shader[] =
#ifdef NANOVG_GL3_IMPLEMENTATION
#ifdef __APPLE__
"#version 150 core\n"
#else
"#version 300 es\n"

3
selfdrive/ui/qt/widgets/cameraview.h
View File

@ -7,6 +7,7 @@
#include <QOpenGLWidget>
#include <QThread>
#include "cereal/visionipc/visionipc_client.h"
#include "selfdrive/camerad/cameras/camera_common.h"
#include "selfdrive/common/visionimg.h"
#include "selfdrive/ui/ui.h"
@ -32,7 +33,7 @@ protected:
void showEvent(QShowEvent *event) override;
void hideEvent(QHideEvent *event) override;
void mouseReleaseEvent(QMouseEvent *event) override { emit clicked(); }
void updateFrameMat(int w, int h);
virtual void updateFrameMat(int w, int h);
void vipcThread();
bool zoomed_view;

2
selfdrive/ui/soundd/sound.cc
View File

@ -1,5 +1,7 @@
#include "selfdrive/ui/soundd/sound.h"
#include <cmath>
#include <QAudio>
#include <QAudioDeviceInfo>
#include <QDebug>

18
selfdrive/ui/ui.cc
View File

@ -1,11 +1,11 @@
#include "selfdrive/ui/ui.h"
#include <unistd.h>
#include <cassert>
#include <cmath>
#include <cstdio>
#include "common/transformations/orientation.hpp"
#include "selfdrive/common/params.h"
#include "selfdrive/common/swaglog.h"
#include "selfdrive/common/util.h"
#include "selfdrive/common/watchdog.h"
#include "selfdrive/hardware/hw.h"
@ -14,10 +14,9 @@
#define BACKLIGHT_TS 10.00
#define BACKLIGHT_OFFROAD 50
// Projects a point in car to space to the corresponding point in full frame
// image space.
static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, float in_z, vertex_data *out) {
static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y, float in_z, QPointF *out) {
const float margin = 500.0f;
const QRectF clip_region{-margin, -margin, s->fb_w + 2 * margin, s->fb_h + 2 * margin};
@ -28,8 +27,7 @@ static bool calib_frame_to_full_frame(const UIState *s, float in_x, float in_y,
// Project.
QPointF point = s->car_space_transform.map(QPointF{KEp.v[0] / KEp.v[2], KEp.v[1] / KEp.v[2]});
if (clip_region.contains(point)) {
out->x = point.x();
out->y = point.y();
*out = point;
return true;
}
return false;
@ -57,7 +55,7 @@ static void update_leads(UIState *s, const cereal::RadarState::Reader &radar_sta
static void update_line_data(const UIState *s, const cereal::ModelDataV2::XYZTData::Reader &line,
float y_off, float z_off, line_vertices_data *pvd, int max_idx) {
const auto line_x = line.getX(), line_y = line.getY(), line_z = line.getZ();
vertex_data *v = &pvd->v[0];
QPointF *v = &pvd->v[0];
for (int i = 0; i <= max_idx; i++) {
v += calib_frame_to_full_frame(s, line_x[i], line_y[i] - y_off, line_z[i] + z_off, v);
}
@ -110,10 +108,10 @@ static void update_state(UIState *s) {
UIScene &scene = s->scene;
s->running_time = 1e-9 * (nanos_since_boot() - sm["deviceState"].getDeviceState().getStartedMonoTime());
if (sm.updated("modelV2") && s->vg) {
if (sm.updated("modelV2")) {
update_model(s, sm["modelV2"].getModelV2());
}
if (sm.updated("radarState") && s->vg) {
if (sm.updated("radarState")) {
std::optional<cereal::ModelDataV2::XYZTData::Reader> line;
if (sm.rcv_frame("modelV2") > 0) {
line = sm["modelV2"].getModelV2().getPosition();

26
selfdrive/ui/ui.h
View File

@ -1,29 +1,17 @@
#pragma once
#include <map>
#include <memory>
#include <string>
#include <optional>
#include <QObject>
#include <QTimer>
#include <QColor>
#include <QTransform>
#include "nanovg.h"
#include "cereal/messaging/messaging.h"
#include "common/transformations/orientation.hpp"
#include "selfdrive/camerad/cameras/camera_common.h"
#include "selfdrive/common/mat.h"
#include "selfdrive/common/modeldata.h"
#include "selfdrive/common/params.h"
#include "selfdrive/common/util.h"
#define COLOR_WHITE nvgRGBA(255, 255, 255, 255)
#define COLOR_WHITE_ALPHA(x) nvgRGBA(255, 255, 255, x)
#define COLOR_RED_ALPHA(x) nvgRGBA(201, 34, 49, x)
#define COLOR_YELLOW nvgRGBA(218, 202, 37, 255)
#define COLOR_RED nvgRGBA(201, 34, 49, 255)
#include "selfdrive/common/timing.h"
const int bdr_s = 30;
const int header_h = 420;
@ -87,16 +75,11 @@ const QColor bg_colors [] = {
};
typedef struct {
float x, y;
} vertex_data;
typedef struct {
vertex_data v[TRAJECTORY_SIZE * 2];
QPointF v[TRAJECTORY_SIZE * 2];
int cnt;
} line_vertices_data;
typedef struct UIScene {
mat3 view_from_calib;
bool world_objects_visible;
@ -110,7 +93,7 @@ typedef struct UIScene {
line_vertices_data road_edge_vertices[2];
// lead
vertex_data lead_vertices[2];
QPointF lead_vertices[2];
float light_sensor, accel_sensor, gyro_sensor;
bool started, ignition, is_metric, longitudinal_control, end_to_end;
@ -119,7 +102,6 @@ typedef struct UIScene {
typedef struct UIState {
int fb_w = 0, fb_h = 0;
NVGcontext *vg;
std::unique_ptr<SubMaster> sm;
@ -189,3 +171,5 @@ public slots:
void setAwake(bool on, bool reset);
void update(const UIState &s);
};
void ui_update_params(UIState *s);