277 lines
8.4 KiB
C++
277 lines
8.4 KiB
C++
#include "selfdrive/ui/qt/widgets/cameraview.h"
|
|
|
|
namespace {
|
|
|
|
const char frame_vertex_shader[] =
|
|
#ifdef NANOVG_GL3_IMPLEMENTATION
|
|
"#version 150 core\n"
|
|
#else
|
|
"#version 300 es\n"
|
|
#endif
|
|
"in vec4 aPosition;\n"
|
|
"in vec4 aTexCoord;\n"
|
|
"uniform mat4 uTransform;\n"
|
|
"out vec4 vTexCoord;\n"
|
|
"void main() {\n"
|
|
" gl_Position = uTransform * aPosition;\n"
|
|
" vTexCoord = aTexCoord;\n"
|
|
"}\n";
|
|
|
|
const char frame_fragment_shader[] =
|
|
#ifdef NANOVG_GL3_IMPLEMENTATION
|
|
"#version 150 core\n"
|
|
#else
|
|
"#version 300 es\n"
|
|
#endif
|
|
"precision mediump float;\n"
|
|
"uniform sampler2D uTexture;\n"
|
|
"in vec4 vTexCoord;\n"
|
|
"out vec4 colorOut;\n"
|
|
"void main() {\n"
|
|
" colorOut = texture(uTexture, vTexCoord.xy);\n"
|
|
#ifdef QCOM
|
|
" vec3 dz = vec3(0.0627f, 0.0627f, 0.0627f);\n"
|
|
" colorOut.rgb = ((vec3(1.0f, 1.0f, 1.0f) - dz) * colorOut.rgb / vec3(1.0f, 1.0f, 1.0f)) + dz;\n"
|
|
#endif
|
|
"}\n";
|
|
|
|
const mat4 device_transform = {{
|
|
1.0, 0.0, 0.0, 0.0,
|
|
0.0, 1.0, 0.0, 0.0,
|
|
0.0, 0.0, 1.0, 0.0,
|
|
0.0, 0.0, 0.0, 1.0,
|
|
}};
|
|
|
|
mat4 get_driver_view_transform() {
|
|
const float driver_view_ratio = 1.333;
|
|
mat4 transform;
|
|
if (Hardware::TICI()) {
|
|
// from dmonitoring.cc
|
|
const int full_width_tici = 1928;
|
|
const int full_height_tici = 1208;
|
|
const int adapt_width_tici = 954;
|
|
const int crop_x_offset = -72;
|
|
const int crop_y_offset = -144;
|
|
const float yscale = full_height_tici * driver_view_ratio / adapt_width_tici;
|
|
const float xscale = yscale*(1080)/(2160)*full_width_tici/full_height_tici;
|
|
transform = (mat4){{
|
|
xscale, 0.0, 0.0, xscale*crop_x_offset/full_width_tici*2,
|
|
0.0, yscale, 0.0, yscale*crop_y_offset/full_height_tici*2,
|
|
0.0, 0.0, 1.0, 0.0,
|
|
0.0, 0.0, 0.0, 1.0,
|
|
}};
|
|
} else {
|
|
// frame from 4/3 to 16/9 display
|
|
transform = (mat4){{
|
|
driver_view_ratio*(1080)/(1920), 0.0, 0.0, 0.0,
|
|
0.0, 1.0, 0.0, 0.0,
|
|
0.0, 0.0, 1.0, 0.0,
|
|
0.0, 0.0, 0.0, 1.0,
|
|
}};
|
|
}
|
|
return transform;
|
|
}
|
|
|
|
mat4 get_fit_view_transform(float widget_aspect_ratio, float frame_aspect_ratio) {
|
|
float zx = 1, zy = 1;
|
|
if (frame_aspect_ratio > widget_aspect_ratio) {
|
|
zy = widget_aspect_ratio / frame_aspect_ratio;
|
|
} else {
|
|
zx = frame_aspect_ratio / widget_aspect_ratio;
|
|
}
|
|
|
|
const mat4 frame_transform = {{
|
|
zx, 0.0, 0.0, 0.0,
|
|
0.0, zy, 0.0, 0.0,
|
|
0.0, 0.0, 1.0, 0.0,
|
|
0.0, 0.0, 0.0, 1.0,
|
|
}};
|
|
return frame_transform;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
CameraViewWidget::CameraViewWidget(VisionStreamType stream_type, bool zoom, QWidget* parent) :
|
|
stream_type(stream_type), zoomed_view(zoom), QOpenGLWidget(parent) {
|
|
setAttribute(Qt::WA_OpaquePaintEvent);
|
|
|
|
QTimer *t = new QTimer(this);
|
|
connect(t, &QTimer::timeout, this, &CameraViewWidget::updateFrame);
|
|
t->start(10);
|
|
}
|
|
|
|
CameraViewWidget::~CameraViewWidget() {
|
|
makeCurrent();
|
|
if (isValid()) {
|
|
glDeleteVertexArrays(1, &frame_vao);
|
|
glDeleteBuffers(1, &frame_vbo);
|
|
glDeleteBuffers(1, &frame_ibo);
|
|
}
|
|
doneCurrent();
|
|
}
|
|
|
|
void CameraViewWidget::initializeGL() {
|
|
initializeOpenGLFunctions();
|
|
|
|
program = new QOpenGLShaderProgram(context());
|
|
bool ret = program->addShaderFromSourceCode(QOpenGLShader::Vertex, frame_vertex_shader);
|
|
assert(ret);
|
|
ret = program->addShaderFromSourceCode(QOpenGLShader::Fragment, frame_fragment_shader);
|
|
assert(ret);
|
|
|
|
program->link();
|
|
GLint frame_pos_loc = program->attributeLocation("aPosition");
|
|
GLint frame_texcoord_loc = program->attributeLocation("aTexCoord");
|
|
|
|
auto [x1, x2, y1, y2] = stream_type == VISION_STREAM_RGB_FRONT ? std::tuple(0.f, 1.f, 1.f, 0.f) : std::tuple(1.f, 0.f, 1.f, 0.f);
|
|
const uint8_t frame_indicies[] = {0, 1, 2, 0, 2, 3};
|
|
const float frame_coords[4][4] = {
|
|
{-1.0, -1.0, x2, y1}, // bl
|
|
{-1.0, 1.0, x2, y2}, // tl
|
|
{ 1.0, 1.0, x1, y2}, // tr
|
|
{ 1.0, -1.0, x1, y1}, // br
|
|
};
|
|
|
|
glGenVertexArrays(1, &frame_vao);
|
|
glBindVertexArray(frame_vao);
|
|
glGenBuffers(1, &frame_vbo);
|
|
glBindBuffer(GL_ARRAY_BUFFER, frame_vbo);
|
|
glBufferData(GL_ARRAY_BUFFER, sizeof(frame_coords), frame_coords, GL_STATIC_DRAW);
|
|
glEnableVertexAttribArray(frame_pos_loc);
|
|
glVertexAttribPointer(frame_pos_loc, 2, GL_FLOAT, GL_FALSE,
|
|
sizeof(frame_coords[0]), (const void *)0);
|
|
glEnableVertexAttribArray(frame_texcoord_loc);
|
|
glVertexAttribPointer(frame_texcoord_loc, 2, GL_FLOAT, GL_FALSE,
|
|
sizeof(frame_coords[0]), (const void *)(sizeof(float) * 2));
|
|
glGenBuffers(1, &frame_ibo);
|
|
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, frame_ibo);
|
|
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(frame_indicies), frame_indicies, GL_STATIC_DRAW);
|
|
glBindBuffer(GL_ARRAY_BUFFER, 0);
|
|
glBindVertexArray(0);
|
|
|
|
setStreamType(stream_type);
|
|
}
|
|
|
|
|
|
void CameraViewWidget::hideEvent(QHideEvent *event) {
|
|
vipc_client->connected = false;
|
|
latest_frame = nullptr;
|
|
}
|
|
|
|
void CameraViewWidget::mouseReleaseEvent(QMouseEvent *event) {
|
|
emit clicked();
|
|
}
|
|
|
|
void CameraViewWidget::resizeGL(int w, int h) {
|
|
updateFrameMat(w, h);
|
|
}
|
|
|
|
void CameraViewWidget::setStreamType(VisionStreamType type) {
|
|
if (!vipc_client || type != stream_type) {
|
|
stream_type = type;
|
|
vipc_client.reset(new VisionIpcClient("camerad", stream_type, true));
|
|
updateFrameMat(width(), height());
|
|
}
|
|
}
|
|
|
|
void CameraViewWidget::setBackgroundColor(QColor color) {
|
|
bg = color;
|
|
}
|
|
|
|
void CameraViewWidget::updateFrameMat(int w, int h) {
|
|
if (zoomed_view) {
|
|
if (stream_type == VISION_STREAM_RGB_FRONT) {
|
|
frame_mat = matmul(device_transform, get_driver_view_transform());
|
|
} else {
|
|
auto intrinsic_matrix = stream_type == VISION_STREAM_RGB_WIDE ? ecam_intrinsic_matrix : fcam_intrinsic_matrix;
|
|
float zoom = ZOOM / intrinsic_matrix.v[0];
|
|
if (stream_type == VISION_STREAM_RGB_WIDE) {
|
|
zoom *= 0.5;
|
|
}
|
|
float zx = zoom * 2 * intrinsic_matrix.v[2] / width();
|
|
float zy = zoom * 2 * intrinsic_matrix.v[5] / height();
|
|
|
|
const mat4 frame_transform = {{
|
|
zx, 0.0, 0.0, 0.0,
|
|
0.0, zy, 0.0, -y_offset / height() * 2,
|
|
0.0, 0.0, 1.0, 0.0,
|
|
0.0, 0.0, 0.0, 1.0,
|
|
}};
|
|
frame_mat = matmul(device_transform, frame_transform);
|
|
}
|
|
} else if (vipc_client->connected) {
|
|
// fit frame to widget size
|
|
float widget_aspect_ratio = (float)width() / height();
|
|
float frame_aspect_ratio = (float)vipc_client->buffers[0].width / vipc_client->buffers[0].height;
|
|
frame_mat = matmul(device_transform, get_fit_view_transform(widget_aspect_ratio, frame_aspect_ratio));
|
|
}
|
|
}
|
|
|
|
void CameraViewWidget::paintGL() {
|
|
if (!latest_frame) {
|
|
glClearColor(bg.redF(), bg.greenF(), bg.blueF(), bg.alphaF());
|
|
glClear(GL_STENCIL_BUFFER_BIT | GL_COLOR_BUFFER_BIT);
|
|
return;
|
|
}
|
|
|
|
glViewport(0, 0, width(), height());
|
|
|
|
glBindVertexArray(frame_vao);
|
|
glActiveTexture(GL_TEXTURE0);
|
|
|
|
glBindTexture(GL_TEXTURE_2D, texture[latest_frame->idx]->frame_tex);
|
|
if (!Hardware::EON()) {
|
|
// this is handled in ion on QCOM
|
|
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, latest_frame->width, latest_frame->height,
|
|
0, GL_RGB, GL_UNSIGNED_BYTE, latest_frame->addr);
|
|
}
|
|
|
|
glUseProgram(program->programId());
|
|
glUniform1i(program->uniformLocation("uTexture"), 0);
|
|
glUniformMatrix4fv(program->uniformLocation("uTransform"), 1, GL_TRUE, frame_mat.v);
|
|
|
|
assert(glGetError() == GL_NO_ERROR);
|
|
glEnableVertexAttribArray(0);
|
|
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, (const void *)0);
|
|
glDisableVertexAttribArray(0);
|
|
glBindVertexArray(0);
|
|
}
|
|
|
|
void CameraViewWidget::updateFrame() {
|
|
if (!isVisible()) {
|
|
return;
|
|
}
|
|
|
|
if (!vipc_client->connected) {
|
|
makeCurrent();
|
|
if (vipc_client->connect(false)) {
|
|
// init vision
|
|
for (int i = 0; i < vipc_client->num_buffers; i++) {
|
|
texture[i].reset(new EGLImageTexture(&vipc_client->buffers[i]));
|
|
|
|
glBindTexture(GL_TEXTURE_2D, texture[i]->frame_tex);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
|
|
|
|
// BGR
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_R, GL_BLUE);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_G, GL_GREEN);
|
|
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_SWIZZLE_B, GL_RED);
|
|
assert(glGetError() == GL_NO_ERROR);
|
|
}
|
|
latest_frame = nullptr;
|
|
resizeGL(width(), height());
|
|
}
|
|
}
|
|
|
|
VisionBuf *buf = nullptr;
|
|
if (vipc_client->connected) {
|
|
buf = vipc_client->recv(nullptr, 0);
|
|
if (buf != nullptr) {
|
|
latest_frame = buf;
|
|
update();
|
|
emit frameUpdated();
|
|
}
|
|
}
|
|
}
|