spacecruft
/
nopenpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

replay: add qcam and YUV flags (#22788) 

* common flags

* cleanup

* remove double semicolon

* camera

* apply reviews
pull/22844/head
Dean Lee 2021-11-10 06:08:24 +08:00 committed by GitHub
parent 6e5fed63df
commit fb8ba34f31
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 43 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
selfdrive/ui/replay/camera.cc
View File

@ -5,9 +5,9 @@
const int YUV_BUF_COUNT = 50;
CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS]) {
for (auto &cam : cameras_) {
std::tie(cam.width, cam.height) = camera_size[cam.type];
CameraServer::CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS], bool send_yuv) : send_yuv(send_yuv) {
for (int i = 0; i < MAX_CAMERAS; ++i) {
std::tie(cameras_[i].width, cameras_[i].height) = camera_size[i];
}
startVipcServer();
}
@ -28,7 +28,9 @@ void CameraServer::startVipcServer() {
if (cam.width > 0 && cam.height > 0) {
std::cout << "camera[" << cam.type << "] frame size " << cam.width << "x" << cam.height << std::endl;
vipc_server_->create_buffers(cam.rgb_type, UI_BUF_COUNT, true, cam.width, cam.height);
vipc_server_->create_buffers(cam.yuv_type, YUV_BUF_COUNT, false, cam.width, cam.height);
if (send_yuv) {
vipc_server_->create_buffers(cam.yuv_type, YUV_BUF_COUNT, false, cam.width, cam.height);
}
if (!cam.thread.joinable()) {
cam.thread = std::thread(&CameraServer::cameraThread, this, std::ref(cam));
}
@ -40,8 +42,8 @@ void CameraServer::startVipcServer() {
void CameraServer::cameraThread(Camera &cam) {
auto read_frame = [&](FrameReader *fr, int frame_id) {
VisionBuf *rgb_buf = vipc_server_->get_buffer(cam.rgb_type);
VisionBuf *yuv_buf = vipc_server_->get_buffer(cam.yuv_type);
bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, (uint8_t *)yuv_buf->addr);
VisionBuf *yuv_buf = send_yuv ? vipc_server_->get_buffer(cam.yuv_type) : nullptr;
bool ret = fr->get(frame_id, (uint8_t *)rgb_buf->addr, yuv_buf ? (uint8_t *)yuv_buf->addr : nullptr);
return ret ? std::pair{rgb_buf, yuv_buf} : std::pair{nullptr, nullptr};
};
@ -50,17 +52,16 @@ void CameraServer::cameraThread(Camera &cam) {
if (!fr) break;
const int id = eidx.getSegmentId();
bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg && cam.cached_buf.first && cam.cached_buf.second);
bool prefetched = (id == cam.cached_id && eidx.getSegmentNum() == cam.cached_seg);
auto [rgb, yuv] = prefetched ? cam.cached_buf : read_frame(fr, id);
if (rgb && yuv) {
if (rgb || yuv) {
VisionIpcBufExtra extra = {
.frame_id = eidx.getFrameId(),
.timestamp_sof = eidx.getTimestampSof(),
.timestamp_eof = eidx.getTimestampEof(),
};
vipc_server_->send(rgb, &extra, false);
vipc_server_->send(yuv, &extra, false);
if (rgb) vipc_server_->send(rgb, &extra, false);
if (yuv) vipc_server_->send(yuv, &extra, false);
} else {
std::cout << "camera[" << cam.type << "] failed to get frame:" << eidx.getSegmentId() << std::endl;
}

3
selfdrive/ui/replay/camera.h
View File

@ -8,7 +8,7 @@
class CameraServer {
public:
CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr);
CameraServer(std::pair<int, int> camera_size[MAX_CAMERAS] = nullptr, bool send_yuv = false);
~CameraServer();
void pushFrame(CameraType type, FrameReader* fr, const cereal::EncodeIndex::Reader& eidx);
inline void waitFinish() {
@ -38,4 +38,5 @@ protected:
};
std::atomic<int> publishing_ = 0;
std::unique_ptr<VisionIpcServer> vipc_server_;
bool send_yuv;
};

16
selfdrive/ui/replay/framereader.cc
View File

@ -31,7 +31,7 @@ int readFunction(void *opaque, uint8_t *buf, int buf_size) {
return iss.gcount() ? iss.gcount() : AVERROR_EOF;
}
} // namespace
} // namespace
FrameReader::FrameReader(bool local_cache, int chunk_size, int retries) : FileReader(local_cache, chunk_size, retries) {
static std::once_flag once_flag;
@ -43,7 +43,7 @@ FrameReader::FrameReader(bool local_cache, int chunk_size, int retries) : FileRe
pFormatCtx_ = avformat_alloc_context();
av_frame_ = av_frame_alloc();
rgb_frame_ = av_frame_alloc();
yuv_frame_ = av_frame_alloc();;
yuv_frame_ = av_frame_alloc();
}
FrameReader::~FrameReader() {
@ -126,7 +126,7 @@ bool FrameReader::load(const std::string &url, std::atomic<bool> *abort) {
}
bool FrameReader::get(int idx, uint8_t *rgb, uint8_t *yuv) {
assert(rgb != nullptr && yuv != nullptr);
assert(rgb || yuv);
if (!valid_ || idx < 0 || idx >= frames_.size()) {
return false;
}
@ -162,11 +162,13 @@ bool FrameReader::decode(int idx, uint8_t *rgb, uint8_t *yuv) {
bool FrameReader::decodeFrame(AVFrame *f, uint8_t *rgb, uint8_t *yuv) {
// images is going to be written to output buffers, no alignment (align = 1)
av_image_fill_arrays(yuv_frame_->data, yuv_frame_->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1);
int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, yuv_frame_->data, yuv_frame_->linesize);
if (ret < 0) return false;
if (yuv) {
av_image_fill_arrays(yuv_frame_->data, yuv_frame_->linesize, yuv, AV_PIX_FMT_YUV420P, width, height, 1);
int ret = sws_scale(yuv_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, yuv_frame_->data, yuv_frame_->linesize);
if (ret < 0) return false;
}
av_image_fill_arrays(rgb_frame_->data, rgb_frame_->linesize, rgb, AV_PIX_FMT_BGR24, width, height, 1);
ret = sws_scale(rgb_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, rgb_frame_->data, rgb_frame_->linesize);
int ret = sws_scale(rgb_sws_ctx_, (const uint8_t **)f->data, f->linesize, 0, f->height, rgb_frame_->data, rgb_frame_->linesize);
return ret >= 0;
}

2
selfdrive/ui/replay/main.cc
View File

@ -100,6 +100,8 @@ int main(int argc, char *argv[]) {
{"ecam", REPLAY_FLAG_ECAM, "load wide road camera"},
{"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"},
{"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"},
{"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"},
{"yuv", REPLAY_FLAG_SEND_YUV, "send yuv frame"},
};
QCommandLineParser parser;

10
selfdrive/ui/replay/replay.cc
View File

@ -151,7 +151,7 @@ void Replay::queueSegment() {
if (!it->second) {
if (it == cur || std::prev(it)->second->isLoaded()) {
auto &[n, seg] = *it;
seg = std::make_unique<Segment>(n, route_->at(n), hasFlag(REPLAY_FLAG_DCAM), hasFlag(REPLAY_FLAG_ECAM), !hasFlag(REPLAY_FLAG_NO_FILE_CACHE));
seg = std::make_unique<Segment>(n, route_->at(n), flags_);
QObject::connect(seg.get(), &Segment::loadFinished, this, &Replay::segmentLoadFinished);
qDebug() << "loading segment" << n << "...";
}
@ -230,7 +230,7 @@ void Replay::startStream(const Segment *cur_segment) {
camera_size[type] = {fr->width, fr->height};
}
}
camera_server_ = std::make_unique<CameraServer>(camera_size);
camera_server_ = std::make_unique<CameraServer>(camera_size, flags_ & REPLAY_FLAG_SEND_YUV);
// start stream thread
stream_thread_ = new QThread();
@ -258,8 +258,8 @@ void Replay::publishFrame(const Event *e) {
{cereal::Event::DRIVER_ENCODE_IDX, DriverCam},
{cereal::Event::WIDE_ROAD_ENCODE_IDX, WideRoadCam},
};
if ((e->which == cereal::Event::DRIVER_ENCODE_IDX && !hasFlag(REPLAY_FLAG_DCAM)) ||
(e->which == cereal::Event::WIDE_ROAD_ENCODE_IDX && !hasFlag(REPLAY_FLAG_ECAM))) {
if ((e->which == cereal::Event::DRIVER_ENCODE_IDX && !(flags_ & REPLAY_FLAG_DCAM)) ||
(e->which == cereal::Event::WIDE_ROAD_ENCODE_IDX && !(flags_ & REPLAY_FLAG_ECAM))) {
return;
}
auto eidx = capnp::AnyStruct::Reader(e->event).getPointerSection()[0].getAs<cereal::EncodeIndex>();
@ -334,7 +334,7 @@ void Replay::stream() {
// wait for frame to be sent before unlock.(frameReader may be deleted after unlock)
camera_server_->waitFinish();
if (eit == events_->end() && !hasFlag(REPLAY_FLAG_NO_LOOP)) {
if (eit == events_->end() && !(flags_ & REPLAY_FLAG_NO_LOOP)) {
int last_segment = segments_.rbegin()->first;
if (current_segment_ >= last_segment && isSegmentMerged(last_segment)) {
qInfo() << "reaches the end of route, restart from beginning";

3
selfdrive/ui/replay/replay.h
View File

@ -14,6 +14,8 @@ enum REPLAY_FLAGS {
REPLAY_FLAG_ECAM = 0x0004,
REPLAY_FLAG_NO_LOOP = 0x0010,
REPLAY_FLAG_NO_FILE_CACHE = 0x0020,
REPLAY_FLAG_QCAMERA = 0x0040,
REPLAY_FLAG_SEND_YUV = 0x0080,
};
class Replay : public QObject {
@ -28,7 +30,6 @@ public:
void stop();
void pause(bool pause);
bool isPaused() const { return paused_; }
inline bool hasFlag(REPLAY_FLAGS flag) { return flags_ & flag; };
signals:
void segmentChanged();

11
selfdrive/ui/replay/route.cc
View File

@ -9,6 +9,7 @@
#include "selfdrive/hardware/hw.h"
#include "selfdrive/ui/qt/api.h"
#include "selfdrive/ui/replay/replay.h"
#include "selfdrive/ui/replay/util.h"
Route::Route(const QString &route, const QString &data_dir) : data_dir_(data_dir) {
@ -90,18 +91,18 @@ void Route::addFileToSegment(int n, const QString &file) {
// class Segment
Segment::Segment(int n, const SegmentFile &files, bool load_dcam, bool load_ecam, bool local_cache) : seg_num(n) {
Segment::Segment(int n, const SegmentFile &files, uint32_t flags) : seg_num(n) {
// [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog
const QString file_list[] = {
files.road_cam.isEmpty() ? files.qcamera : files.road_cam,
load_dcam ? files.driver_cam : "",
load_ecam ? files.wide_road_cam : "",
(flags & REPLAY_FLAG_QCAMERA) || files.road_cam.isEmpty() ? files.qcamera : files.road_cam,
flags & REPLAY_FLAG_DCAM ? files.driver_cam : "",
flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "",
files.rlog.isEmpty() ? files.qlog : files.rlog,
};
for (int i = 0; i < std::size(file_list); i++) {
if (!file_list[i].isEmpty()) {
loading_++;
synchronizer_.addFuture(QtConcurrent::run([=] { loadFile(i, file_list[i].toStdString(), local_cache); }));
synchronizer_.addFuture(QtConcurrent::run([=] { loadFile(i, file_list[i].toStdString(), !(flags & REPLAY_FLAG_NO_FILE_CACHE)); }));
}
}
}

5
selfdrive/ui/replay/route.h
View File

@ -4,6 +4,7 @@
#include "selfdrive/ui/replay/framereader.h"
#include "selfdrive/ui/replay/logreader.h"
#include "selfdrive/ui/replay/util.h"
struct RouteIdentifier {
QString dongle_id;
@ -28,7 +29,7 @@ public:
inline const QString &name() const { return route_.str; }
inline const RouteIdentifier &identifier() const { return route_; }
inline const std::map<int, SegmentFile> &segments() const { return segments_; }
inline SegmentFile &at(int n) { return segments_.at(n); }
inline const SegmentFile &at(int n) { return segments_.at(n); }
static RouteIdentifier parseRoute(const QString &str);
protected:
@ -45,7 +46,7 @@ class Segment : public QObject {
Q_OBJECT
public:
Segment(int n, const SegmentFile &files, bool load_dcam, bool load_ecam, bool local_cache);
Segment(int n, const SegmentFile &files, uint32_t flags);
~Segment();
inline bool isLoaded() const { return !loading_ && !abort_; }

8
selfdrive/ui/replay/tests/test_replay.cc
View File

@ -57,17 +57,13 @@ TEST_CASE("FileReader") {
}
TEST_CASE("Segment") {
auto test_qlog = GENERATE(false, true);
auto flags = GENERATE(REPLAY_FLAG_NONE, REPLAY_FLAG_QCAMERA);
Route demo_route(DEMO_ROUTE);
REQUIRE(demo_route.load());
REQUIRE(demo_route.segments().size() == 11);
if (test_qlog) {
demo_route.at(0).road_cam = "";
demo_route.at(0).rlog = "";
}
QEventLoop loop;
Segment segment(0, demo_route.at(0), false, false, false);
Segment segment(0, demo_route.at(0), flags);
QObject::connect(&segment, &Segment::loadFinished, [&]() {
REQUIRE(segment.isLoaded() == true);
REQUIRE(segment.log != nullptr);