From 19d962cdf37b80523deba6518057f2e860f65fee Mon Sep 17 00:00:00 2001 From: iejMac <61431446+iejMac@users.noreply.github.com> Date: Sat, 24 Apr 2021 00:59:09 -0700 Subject: [PATCH] qt replay (#20602) * initial commit, works * remove nui * working again * visionipc * cleanup * cleanup * moving VisionIpcServer to Unlogger class * works * tab cleanup * headless mode * headless mode works * working headless mode * gitignore update * small unlogger refactor * refactor param in UIState * works, very slow, hacks * cleanup * works * cleanup * cleanup * unused * works for whole route * nicer * a little nicer * different threshold * maintains 1 segment window * works with public api * comments * networkTimer works * cleanup * unified HttpRequest * tabs * tabs * comments' * gitignore * gitignore * only on PC * same line else * no changes in home.cc * scons * update scons * works * revert mainc.c * revert home * else * just api + problem with api send * works * include cleanup * general json fail * whitespace * remove active * adding request repeater * removing comments * tabs * update comment * cereal * fix * trailing new lines * grammar * if whitespace * indentation * Update selfdrive/ui/SConscript Co-authored-by: Adeeb Shihadeh * Update selfdrive/ui/qt/request_repeater.cc Co-authored-by: Adeeb Shihadeh * works * sort by dir * no blockSignal * replay is now QOBject * cant take const char * rename inner it * get width and height from frame readeR * resolve TODO * seek in next pr * spaces * ui stuff * fix CI * remove comments * no repalce * trim segment fix * remove seek from stream * no cache key * final changes' * fix Co-authored-by: Adeeb Shihadeh --- SConstruct | 3 - selfdrive/ui/.gitignore | 1 + selfdrive/ui/SConscript | 22 +- selfdrive/ui/qt/api.cc | 24 +- selfdrive/ui/qt/api.hpp | 17 +- selfdrive/ui/qt/request_repeater.cc | 12 + selfdrive/ui/qt/request_repeater.hpp | 9 + selfdrive/ui/qt/widgets/drive_stats.cc | 4 +- selfdrive/ui/qt/widgets/setup.cc | 6 +- .../ui/replay}/FileReader.cpp | 0 .../ui/replay}/FileReader.hpp | 0 .../nui => selfdrive/ui/replay}/Unlogger.cpp | 99 ++++--- .../nui => selfdrive/ui/replay}/Unlogger.hpp | 11 +- selfdrive/ui/replay/main.cc | 18 ++ selfdrive/ui/replay/replay.cc | 71 +++++ selfdrive/ui/replay/replay.hpp | 48 ++++ tools/clib/FrameReader.hpp | 8 +- tools/nui/.gitignore | 8 - tools/nui/SConscript | 11 - tools/nui/get_files_comma_api.py | 14 - tools/nui/main.cpp | 262 ------------------ tools/nui/nui | 18 -- tools/nui/test/.gitignore | 1 - tools/nui/test/TestFrameReader.cpp | 14 - tools/nui/test/TestFrameReader.hpp | 8 - tools/nui/test/test.pro | 16 -- 26 files changed, 266 insertions(+), 439 deletions(-) create mode 100644 selfdrive/ui/qt/request_repeater.cc create mode 100644 selfdrive/ui/qt/request_repeater.hpp rename {tools/nui => selfdrive/ui/replay}/FileReader.cpp (100%) rename {tools/nui => selfdrive/ui/replay}/FileReader.hpp (100%) rename {tools/nui => selfdrive/ui/replay}/Unlogger.cpp (63%) rename {tools/nui => selfdrive/ui/replay}/Unlogger.hpp (77%) create mode 100644 selfdrive/ui/replay/main.cc create mode 100644 selfdrive/ui/replay/replay.cc create mode 100644 selfdrive/ui/replay/replay.hpp delete mode 100644 tools/nui/.gitignore delete mode 100644 tools/nui/SConscript delete mode 100755 tools/nui/get_files_comma_api.py delete mode 100644 tools/nui/main.cpp delete mode 100755 tools/nui/nui delete mode 100644 tools/nui/test/.gitignore delete mode 100644 tools/nui/test/TestFrameReader.cpp delete mode 100644 tools/nui/test/TestFrameReader.hpp delete mode 100644 tools/nui/test/test.pro diff --git a/SConstruct b/SConstruct index d0ac47c49..a865690fb 100644 --- a/SConstruct +++ b/SConstruct @@ -417,9 +417,6 @@ SConscript(['selfdrive/ui/SConscript']) if arch != "Darwin": SConscript(['selfdrive/logcatd/SConscript']) -if real_arch == "x86_64": - SConscript(['tools/nui/SConscript']) - external_sconscript = GetOption('external_sconscript') if external_sconscript: SConscript([external_sconscript]) diff --git a/selfdrive/ui/.gitignore b/selfdrive/ui/.gitignore index 63f85bac0..2dbc32523 100644 --- a/selfdrive/ui/.gitignore +++ b/selfdrive/ui/.gitignore @@ -1,6 +1,7 @@ moc_* *.moc +replay/replay qt/text qt/spinner qt/setup/setup diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index e0bb3ecba..92ba34960 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -14,7 +14,8 @@ if arch == "Darwin": widgets_src = ["qt/widgets/input.cc", "qt/widgets/drive_stats.cc", "qt/sound.cc", "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc", "qt/widgets/offroad_alerts.cc", "qt/widgets/setup.cc", "qt/widgets/keyboard.cc", - "qt/widgets/scrollview.cc", "#phonelibs/qrcode/QrCode.cc"] + "qt/widgets/scrollview.cc", "#phonelibs/qrcode/QrCode.cc", "qt/api.cc", + "qt/request_repeater.cc"] if arch != 'aarch64': widgets_src += ["qt/offroad/networking.cc", "qt/offroad/wifiManager.cc"] @@ -26,9 +27,10 @@ qt_env.Program("qt/text", ["qt/text.cc"], LIBS=qt_libs) qt_env.Program("qt/spinner", ["qt/spinner.cc"], LIBS=base_libs) # build main UI -qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc", "#phonelibs/nanovg/nanovg.c", - "qt/window.cc", "qt/home.cc", "qt/api.cc", "qt/offroad/settings.cc", - "qt/offroad/onboarding.cc"] +qt_src = ["main.cc", "ui.cc", "paint.cc", "sidebar.cc", + "qt/window.cc", "qt/home.cc", "qt/offroad/settings.cc", + "qt/offroad/onboarding.cc", "#phonelibs/nanovg/nanovg.c"] + qt_env.Program("_ui", qt_src, LIBS=qt_libs) # setup, factory resetter, and installer @@ -55,3 +57,15 @@ if arch != 'aarch64' and "BUILD_SETUP" in os.environ: d['SSH_KEYS'] = f'\\"{r.text.strip()}\\"' obj = qt_env.Object(f"qt/setup/installer_{name}.o", ["qt/setup/installer.cc"], CPPDEFINES=d) qt_env.Program(f"qt/setup/installer_{name}", obj, LIBS=qt_libs, CPPDEFINES=d) + +# build headless replay +if arch == 'x86_64': + qt_env['CPPPATH'] += ["#tools/clib"] + qt_env['CXXFLAGS'] += ["-Wno-deprecated-declarations"] + + replay_lib_src = ["replay/replay.cc", "replay/Unlogger.cpp", + "replay/FileReader.cpp", "#tools/clib/FrameReader.cpp"] + + replay_lib = qt_env.Library("qt_replay", replay_lib_src, LIBS=base_libs) + replay_libs = [replay_lib, 'avutil', 'avcodec', 'avformat', 'swscale', 'bz2'] + qt_libs + qt_env.Program("replay/replay", ["replay/main.cc"], LIBS=replay_libs) diff --git a/selfdrive/ui/qt/api.cc b/selfdrive/ui/qt/api.cc index 097c1cd5c..eb4603b5d 100644 --- a/selfdrive/ui/qt/api.cc +++ b/selfdrive/ui/qt/api.cc @@ -11,7 +11,6 @@ #include #include "api.hpp" -#include "home.hpp" #include "common/params.h" #include "common/util.h" @@ -72,21 +71,18 @@ QString CommaApi::create_jwt(QVector> payloads, int e return jwt; } -RequestRepeater::RequestRepeater(QWidget* parent, QString requestURL, int period_seconds, const QString &cache_key, bool disableWithScreen) - : disableWithScreen(disableWithScreen), cache_key(cache_key), QObject(parent) { + +HttpRequest::HttpRequest(QObject *parent, QString requestURL, const QString &cache_key) : cache_key(cache_key), QObject(parent) { networkAccessManager = new QNetworkAccessManager(this); - reply = NULL; - QTimer* timer = new QTimer(this); - QObject::connect(timer, &QTimer::timeout, [=](){sendRequest(requestURL);}); - timer->start(period_seconds * 1000); - networkTimer = new QTimer(this); networkTimer->setSingleShot(true); networkTimer->setInterval(20000); connect(networkTimer, SIGNAL(timeout()), this, SLOT(requestTimeout())); + sendRequest(requestURL); + if (!cache_key.isEmpty()) { if (std::string cached_resp = Params().get(cache_key.toStdString()); !cached_resp.empty()) { QTimer::singleShot(0, [=]() { emit receivedResponse(QString::fromStdString(cached_resp)); }); @@ -94,12 +90,7 @@ RequestRepeater::RequestRepeater(QWidget* parent, QString requestURL, int period } } -void RequestRepeater::sendRequest(QString requestURL){ - if (GLWindow::ui_state.scene.started || !active || reply != NULL || - (!GLWindow::ui_state.awake && disableWithScreen)) { - return; - } - +void HttpRequest::sendRequest(QString requestURL){ QString token = CommaApi::create_jwt(); QNetworkRequest request; request.setUrl(QUrl(requestURL)); @@ -118,15 +109,16 @@ void RequestRepeater::sendRequest(QString requestURL){ connect(reply, SIGNAL(finished()), this, SLOT(requestFinished())); } -void RequestRepeater::requestTimeout(){ +void HttpRequest::requestTimeout(){ reply->abort(); } // This function should always emit something -void RequestRepeater::requestFinished(){ +void HttpRequest::requestFinished(){ if (reply->error() != QNetworkReply::OperationCanceledError) { networkTimer->stop(); QString response = reply->readAll(); + if (reply->error() == QNetworkReply::NoError) { // save to cache if (!cache_key.isEmpty()) { diff --git a/selfdrive/ui/qt/api.hpp b/selfdrive/ui/qt/api.hpp index e8e8312cc..5a01946c8 100644 --- a/selfdrive/ui/qt/api.hpp +++ b/selfdrive/ui/qt/api.hpp @@ -26,22 +26,21 @@ private: }; /** - * Makes repeated requests to the request endpoint. + * Makes a request to the request endpoint. */ -class RequestRepeater : public QObject { + +class HttpRequest : public QObject { Q_OBJECT public: - explicit RequestRepeater(QWidget* parent, QString requestURL, int period = 10, const QString &cache_key = "", bool disableWithScreen = true); - bool active = true; + explicit HttpRequest(QObject* parent, QString requestURL, const QString &cache_key = ""); + QNetworkReply *reply; + void sendRequest(QString requestURL); private: - bool disableWithScreen; - QNetworkReply* reply; - QNetworkAccessManager* networkAccessManager; - QTimer* networkTimer; + QNetworkAccessManager *networkAccessManager; + QTimer *networkTimer; QString cache_key; - void sendRequest(QString requestURL); private slots: void requestTimeout(); diff --git a/selfdrive/ui/qt/request_repeater.cc b/selfdrive/ui/qt/request_repeater.cc new file mode 100644 index 000000000..6a5844bf4 --- /dev/null +++ b/selfdrive/ui/qt/request_repeater.cc @@ -0,0 +1,12 @@ +#include "request_repeater.hpp" + +RequestRepeater::RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key, int period_seconds, bool disableWithScreen) : + HttpRequest(parent, requestURL, cache_key), disableWithScreen(disableWithScreen) { + QTimer* timer = new QTimer(this); + QObject::connect(timer, &QTimer::timeout, [=](){ + if (!GLWindow::ui_state.scene.started && reply == NULL && (GLWindow::ui_state.awake || !disableWithScreen)) { + sendRequest(requestURL); + } + }); + timer->start(period_seconds * 1000); +} diff --git a/selfdrive/ui/qt/request_repeater.hpp b/selfdrive/ui/qt/request_repeater.hpp new file mode 100644 index 000000000..e544e580d --- /dev/null +++ b/selfdrive/ui/qt/request_repeater.hpp @@ -0,0 +1,9 @@ +#include "api.hpp" +#include "home.hpp" + +class RequestRepeater : public HttpRequest { + +public: + RequestRepeater(QObject *parent, QString requestURL, const QString &cache_key = "", int period_seconds = 0, bool disableWithScreen = true); + bool disableWithScreen; +}; diff --git a/selfdrive/ui/qt/widgets/drive_stats.cc b/selfdrive/ui/qt/widgets/drive_stats.cc index 83fca7a74..5db55d7b8 100644 --- a/selfdrive/ui/qt/widgets/drive_stats.cc +++ b/selfdrive/ui/qt/widgets/drive_stats.cc @@ -3,9 +3,9 @@ #include #include -#include "api.hpp" #include "common/params.h" #include "drive_stats.hpp" +#include "request_repeater.hpp" const double MILE_TO_KM = 1.60934; @@ -65,6 +65,6 @@ DriveStats::DriveStats(QWidget* parent) : QWidget(parent) { // TODO: do we really need to update this frequently? QString dongleId = QString::fromStdString(Params().get("DongleId")); QString url = "https://api.commadotai.com/v1.1/devices/" + dongleId + "/stats"; - RequestRepeater* repeater = new RequestRepeater(this, url, 13, "ApiCache_DriveStats"); + RequestRepeater *repeater = new RequestRepeater(this, url, "ApiCache_DriveStats", 13); QObject::connect(repeater, SIGNAL(receivedResponse(QString)), this, SLOT(parseResponse(QString))); } diff --git a/selfdrive/ui/qt/widgets/setup.cc b/selfdrive/ui/qt/widgets/setup.cc index c90e0916a..a0480ed1a 100644 --- a/selfdrive/ui/qt/widgets/setup.cc +++ b/selfdrive/ui/qt/widgets/setup.cc @@ -8,7 +8,7 @@ #include #include "QrCode.hpp" -#include "api.hpp" +#include "request_repeater.hpp" #include "common/params.h" #include "setup.hpp" @@ -104,7 +104,7 @@ PrimeUserWidget::PrimeUserWidget(QWidget* parent) : QWidget(parent) { } QString url = "https://api.commadotai.com/v1/devices/" + dongleId + "/owner"; - RequestRepeater* repeater = new RequestRepeater(this, url, 6, "ApiCache_Owner"); + RequestRepeater *repeater = new RequestRepeater(this, url, "ApiCache_Owner", 6); QObject::connect(repeater, SIGNAL(receivedResponse(QString)), this, SLOT(replyFinished(QString))); } @@ -236,7 +236,7 @@ SetupWidget::SetupWidget(QWidget* parent) : QFrame(parent) { // set up API requests QString dongleId = QString::fromStdString(Params().get("DongleId")); QString url = "https://api.commadotai.com/v1.1/devices/" + dongleId + "/"; - RequestRepeater* repeater = new RequestRepeater(this, url, 5, "ApiCache_Device"); + RequestRepeater* repeater = new RequestRepeater(this, url, "ApiCache_Device", 5); QObject::connect(repeater, SIGNAL(receivedResponse(QString)), this, SLOT(replyFinished(QString))); QObject::connect(repeater, SIGNAL(failedResponse(QString)), this, SLOT(parseError(QString))); diff --git a/tools/nui/FileReader.cpp b/selfdrive/ui/replay/FileReader.cpp similarity index 100% rename from tools/nui/FileReader.cpp rename to selfdrive/ui/replay/FileReader.cpp diff --git a/tools/nui/FileReader.hpp b/selfdrive/ui/replay/FileReader.hpp similarity index 100% rename from tools/nui/FileReader.hpp rename to selfdrive/ui/replay/FileReader.hpp diff --git a/tools/nui/Unlogger.cpp b/selfdrive/ui/replay/Unlogger.cpp similarity index 63% rename from tools/nui/Unlogger.cpp rename to selfdrive/ui/replay/Unlogger.cpp index 6455750d7..d789e4132 100644 --- a/tools/nui/Unlogger.cpp +++ b/selfdrive/ui/replay/Unlogger.cpp @@ -14,16 +14,7 @@ #include #include -static inline uint64_t nanos_since_boot() { - struct timespec t; - #ifdef __APPLE__ - clock_gettime(CLOCK_REALTIME, &t); - #else - clock_gettime(CLOCK_BOOTTIME, &t); - #endif - return t.tv_sec * 1000000000ULL + t.tv_nsec; -} - +#include "common/timing.h" Unlogger::Unlogger(Events *events_, QReadWriteLock* events_lock_, QMap *frs_, int seek) : events(events_), events_lock(events_lock_), frs(frs_) { @@ -57,23 +48,12 @@ Unlogger::Unlogger(Events *events_, QReadWriteLock* events_lock_, QMap().getFields()) { - std::string tname = field.getProto().getName(); - - if (tname == name) { - // TODO: I couldn't figure out how to get the which, only the index, hence this hack - int type = field.getIndex(); - if (type > 67) type--; // valid - type--; // logMonoTime - - //qDebug() << "here" << tname.c_str() << type << cereal::Event::CONTROLS_STATE; - socks.insert(type, sock); - } - } + socks.insert(name, sock); } } -void Unlogger::process() { +void Unlogger::process(SubMaster *sm) { + qDebug() << "hello from unlogger thread"; while (events->size() == 0) { qDebug() << "waiting for events"; @@ -103,6 +83,12 @@ void Unlogger::process() { auto eit = events->lowerBound(t0); while (eit != events->end()) { + + float time_to_end = ((events->lastKey() - eit.key())/1e9); + if (loading_segment && (time_to_end > 20.0)){ + loading_segment = false; + } + while (paused) { QThread::usleep(1000); t0 = eit->getLogMonoTime(); @@ -127,8 +113,14 @@ void Unlogger::process() { last_elapsed = tc; } - auto e = *eit; - auto type = e.which(); + cereal::Event::Reader e = *eit; + + capnp::DynamicStruct::Reader e_ds = static_cast(e); + std::string type; + KJ_IF_MAYBE(e_, e_ds.which()){ + type = e_->getProto().getName(); + } + uint64_t tm = e.getLogMonoTime(); auto it = socks.find(type); tc = tm; @@ -147,39 +139,56 @@ void Unlogger::process() { //qDebug() << "sleeping" << us_behind << etime << timer.nsecsElapsed(); } - capnp::MallocMessageBuilder msg; - msg.setRoot(e); + if (type == "roadCameraState") { + auto fr = e.getRoadCameraState(); - auto ee = msg.getRoot(); - ee.setLogMonoTime(nanos_since_boot()); - - if (e.which() == cereal::Event::ROAD_CAMERA_STATE) { - auto fr = msg.getRoot().getRoadCameraState(); - - // TODO: better way? - auto it = eidx.find(fr.getFrameId()); - if (it != eidx.end()) { - auto pp = *it; + auto it_ = eidx.find(fr.getFrameId()); + if (it_ != eidx.end()) { + auto pp = *it_; //qDebug() << fr.getRoadCameraStateId() << pp; if (frs->find(pp.first) != frs->end()) { auto frm = (*frs)[pp.first]; auto data = frm->get(pp.second); - if (data != NULL) { - fr.setImage(kj::arrayPtr(data, frm->getRGBSize())); + + if (vipc_server == nullptr) { + cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT); + cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err)); + + vipc_server = new VisionIpcServer("camerad", device_id, context); + vipc_server->create_buffers(VisionStreamType::VISION_STREAM_RGB_BACK, 4, true, frm->width, frm->height); + + vipc_server->start_listener(); } + + VisionBuf *buf = vipc_server->get_buffer(VisionStreamType::VISION_STREAM_RGB_BACK); + memcpy(buf->addr, data, frm->getRGBSize()); + VisionIpcBufExtra extra = {}; + + vipc_server->send(buf, &extra, false); } } } - auto words = capnp::messageToFlatArray(msg); - auto bytes = words.asBytes(); + if (sm == nullptr){ + capnp::MallocMessageBuilder msg; + msg.setRoot(e); + auto words = capnp::messageToFlatArray(msg); + auto bytes = words.asBytes(); - // TODO: Can PubSocket take a const char? - (*it)->send((char*)bytes.begin(), bytes.size()); + (*it)->send((char*)bytes.begin(), bytes.size()); + } else{ + std::vector> messages; + messages.push_back({type, e}); + sm->update_msgs(nanos_since_boot(), messages); + } } ++eit; + + if (time_to_end < 10.0 && !loading_segment){ + loading_segment = true; + emit loadSegment(); + } } } } - diff --git a/tools/nui/Unlogger.hpp b/selfdrive/ui/replay/Unlogger.hpp similarity index 77% rename from tools/nui/Unlogger.hpp rename to selfdrive/ui/replay/Unlogger.hpp index b77a0d864..bfb091287 100644 --- a/tools/nui/Unlogger.hpp +++ b/selfdrive/ui/replay/Unlogger.hpp @@ -2,9 +2,11 @@ #include #include +#include "clutil.h" #include "messaging.hpp" #include "FileReader.hpp" #include "FrameReader.hpp" +#include "visionipc_server.h" class Unlogger : public QObject { Q_OBJECT @@ -15,19 +17,24 @@ Q_OBJECT void setPause(bool pause) { paused = pause; } void togglePause() { paused = !paused; } QMap > eidx; + public slots: - void process(); + void process(SubMaster *sm = nullptr); signals: void elapsed(); void finished(); + void loadSegment(); private: Events *events; QReadWriteLock *events_lock; QMap *frs; - QMap socks; + QMap socks; Context *ctx; uint64_t tc = 0; uint64_t seek_request = 0; bool paused = false; + bool loading_segment = false; + + VisionIpcServer *vipc_server = nullptr; }; diff --git a/selfdrive/ui/replay/main.cc b/selfdrive/ui/replay/main.cc new file mode 100644 index 000000000..f7db95c01 --- /dev/null +++ b/selfdrive/ui/replay/main.cc @@ -0,0 +1,18 @@ +#include + +#include "replay.hpp" + +int main(int argc, char *argv[]){ + QApplication a(argc, argv); + + QString route(argv[1]); + if (route == "") { + printf("Usage: ./replay \"route\"\n"); + return 1; + } + + Replay *replay = new Replay(route, 0); + replay->stream(0); + + return a.exec(); +} diff --git a/selfdrive/ui/replay/replay.cc b/selfdrive/ui/replay/replay.cc new file mode 100644 index 000000000..8c952ee6b --- /dev/null +++ b/selfdrive/ui/replay/replay.cc @@ -0,0 +1,71 @@ +#include "replay.hpp" + +Replay::Replay(QString route_, int seek) : route(route_) { + unlogger = new Unlogger(&events, &events_lock, &frs, seek); + current_segment = 0; + + http = new HttpRequest(this, "https://api.commadotai.com/v1/route/" + route + "/files"); + QObject::connect(http, SIGNAL(receivedResponse(QString)), this, SLOT(parseResponse(QString))); +} + +void Replay::parseResponse(QString response){ + response = response.trimmed(); + QJsonDocument doc = QJsonDocument::fromJson(response.toUtf8()); + + if (doc.isNull()) { + qDebug() << "JSON Parse failed"; + return; + } + + camera_paths = doc["cameras"].toArray(); + log_paths = doc["logs"].toArray(); + + // add first segment + addSegment(0); +} + +void Replay::addSegment(int i){ + if (lrs.find(i) != lrs.end()) { + return; + } + + QThread* thread = new QThread; + + QString log_fn = this->log_paths.at(i).toString(); + lrs.insert(i, new LogReader(log_fn, &events, &events_lock, &unlogger->eidx)); + + lrs[i]->moveToThread(thread); + QObject::connect(thread, SIGNAL (started()), lrs[i], SLOT (process())); + thread->start(); + + QString camera_fn = this->camera_paths.at(i).toString(); + frs.insert(i, new FrameReader(qPrintable(camera_fn))); +} + +void Replay::trimSegment(int n){ + event_sizes.enqueue(events.size() - event_sizes.last()); + auto first = events.begin(); + + for(int i = 0 ; i < n ; i++){ + int remove = event_sizes.dequeue(); + for(int j = 0 ; j < remove ; j++){ + first = events.erase(first); + } + } +} + +void Replay::stream(SubMaster *sm){ + QThread* thread = new QThread; + unlogger->moveToThread(thread); + QObject::connect(thread, &QThread::started, [=](){ + unlogger->process(sm); + }); + thread->start(); + + QObject::connect(unlogger, &Unlogger::loadSegment, [=](){ + addSegment(++current_segment); + if (current_segment > 1) { + trimSegment(1); + } + }); +} diff --git a/selfdrive/ui/replay/replay.hpp b/selfdrive/ui/replay/replay.hpp new file mode 100644 index 000000000..e6d54ac36 --- /dev/null +++ b/selfdrive/ui/replay/replay.hpp @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "qt/api.hpp" +#include "Unlogger.hpp" +#include "FileReader.hpp" +#include "FrameReader.hpp" +#include "visionipc_server.h" + +class Replay : public QObject { + Q_OBJECT + +public: + Replay(QString route_, int seek); + void stream(SubMaster *sm = nullptr); + void addSegment(int i); + void trimSegment(int n); + QJsonArray camera_paths; + QJsonArray log_paths; + + QQueue event_sizes; + +public slots: + void parseResponse(QString response); + +protected: + Unlogger *unlogger; + +private: + QString route; + + QReadWriteLock events_lock; + Events events; + + QMap lrs; + QMap frs; + HttpRequest *http; + + int current_segment; +}; + diff --git a/tools/clib/FrameReader.hpp b/tools/clib/FrameReader.hpp index db655d641..472444a78 100644 --- a/tools/clib/FrameReader.hpp +++ b/tools/clib/FrameReader.hpp @@ -30,15 +30,17 @@ public: int getRGBSize() { return width*height*3; } void loaderThread(); void cacherThread(); + + //TODO: get this from the actual frame + int width = 1164; + int height = 874; + private: AVFormatContext *pFormatCtx = NULL; AVCodecContext *pCodecCtx = NULL; struct SwsContext *sws_ctx = NULL; - int width = 1164; - int height = 874; - std::vector pkts; std::thread *t; diff --git a/tools/nui/.gitignore b/tools/nui/.gitignore deleted file mode 100644 index 952878aca..000000000 --- a/tools/nui/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.*.swp -*.o -_nui -moc_* -nui.app/* -routes.json -_nui.app - diff --git a/tools/nui/SConscript b/tools/nui/SConscript deleted file mode 100644 index f3d89f4d7..000000000 --- a/tools/nui/SConscript +++ /dev/null @@ -1,11 +0,0 @@ -Import('qt_env', 'messaging') - -qt_env['CPPPATH'] += ["#tools/clib"] -qt_env['CXXFLAGS'] += ["-Wno-deprecated-declarations"] - -libs = [messaging, 'avutil', 'avcodec', 'avformat', 'bz2', 'capnp', 'kj', - 'pthread', 'swscale', 'zmq'] - -qt_env.Program("_nui", - ['main.cpp', 'Unlogger.cpp', 'FileReader.cpp', '../clib/FrameReader.cpp'], - LIBS=qt_env['LIBS'] + libs) diff --git a/tools/nui/get_files_comma_api.py b/tools/nui/get_files_comma_api.py deleted file mode 100755 index b350e8488..000000000 --- a/tools/nui/get_files_comma_api.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -import json -import sys - -from tools.lib.route import Route - -route_name = sys.argv[1] -routes = Route(route_name) -data_dump = { - "camera": routes.camera_paths(), - "logs": routes.log_paths() -} - -json.dump(data_dump, open("routes.json", "w")) diff --git a/tools/nui/main.cpp b/tools/nui/main.cpp deleted file mode 100644 index be661afe5..000000000 --- a/tools/nui/main.cpp +++ /dev/null @@ -1,262 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "FileReader.hpp" -#include "Unlogger.hpp" -#include "FrameReader.hpp" - -class Window : public QWidget { - public: - Window(QString route_, int seek, int use_api); - bool addSegment(int i); - QJsonArray camera_paths; - QJsonArray log_paths; - int use_api; - protected: - void keyPressEvent(QKeyEvent *event) override; - void mousePressEvent(QMouseEvent *event) override; - void paintEvent(QPaintEvent *event) override; - uint64_t ct; - Unlogger *unlogger; - private: - int timeToPixel(uint64_t ns); - uint64_t pixelToTime(int px); - QString route; - - QReadWriteLock events_lock; - Events events; - int last_event_size = 0; - - QMap lrs; - QMap frs; - - - // cache the bar - QPixmap *px = NULL; - int seg_add = 0; - - QLineEdit *timeLE; -}; - -Window::Window(QString route_, int seek, int use_api_) : route(route_), use_api(use_api_) { - timeLE = new QLineEdit(this); - timeLE->setPlaceholderText("Placeholder Text"); - timeLE->move(50, 650); - - QThread* thread = new QThread; - unlogger = new Unlogger(&events, &events_lock, &frs, seek); - unlogger->moveToThread(thread); - connect(thread, SIGNAL (started()), unlogger, SLOT (process())); - connect(unlogger, SIGNAL (elapsed()), this, SLOT (update())); - thread->start(); - - if (use_api) { - QString settings; - QFile file; - file.setFileName("routes.json"); - file.open(QIODevice::ReadOnly | QIODevice::Text); - settings = file.readAll(); - file.close(); - - QJsonDocument sd = QJsonDocument::fromJson(settings.toUtf8()); - qWarning() << sd.isNull(); // <- print false :) - QJsonObject sett2 = sd.object(); - - this->camera_paths = sett2.value("camera").toArray(); - this->log_paths = sett2.value("logs").toArray(); - } - - this->setFocusPolicy(Qt::StrongFocus); - - // add the first segment - addSegment(seek/60); -} - -bool Window::addSegment(int i) { - if (lrs.find(i) == lrs.end()) { - QString fn = QString("http://data.comma.life/%1/%2/rlog.bz2").arg(route).arg(i); - - - QThread* thread = new QThread; - if (!use_api) { - lrs.insert(i, new LogReader(fn, &events, &events_lock, &unlogger->eidx)); - } else { - QString log_fn = this->log_paths.at(i).toString(); - lrs.insert(i, new LogReader(log_fn, &events, &events_lock, &unlogger->eidx)); - - } - - lrs[i]->moveToThread(thread); - connect(thread, SIGNAL (started()), lrs[i], SLOT (process())); - thread->start(); - //connect(lrs[i], SIGNAL (finished()), this, SLOT (update())); - - QString frn = QString("http://data.comma.life/%1/%2/fcamera.hevc").arg(route).arg(i); - - if (!use_api) { - frs.insert(i, new FrameReader(qPrintable(frn))); - } else { - QString camera_fn = this->camera_paths.at(i).toString(); - frs.insert(i, new FrameReader(qPrintable(camera_fn))); - } - - - return true; - } - return false; -} - -#define PIXELS_PER_SEC 0.5 - -int Window::timeToPixel(uint64_t ns) { - // TODO: make this dynamic - return int(ns*1e-9*PIXELS_PER_SEC+0.5); -} - -uint64_t Window::pixelToTime(int px) { - // TODO: make this dynamic - //printf("%d\n", px); - return ((px+0.5)/PIXELS_PER_SEC) * 1e9; -} - -void Window::keyPressEvent(QKeyEvent *event) { - printf("keypress: %x\n", event->key()); - if (event->key() == Qt::Key_Space) unlogger->togglePause(); -} - -void Window::mousePressEvent(QMouseEvent *event) { - //printf("mouse event\n"); - if (event->button() == Qt::LeftButton) { - uint64_t t0 = events.begin().key(); - uint64_t tt = pixelToTime(event->x()); - int seg = int((tt*1e-9)/60); - printf("segment %d\n", seg); - addSegment(seg); - - //printf("seek to %lu\n", t0+tt); - unlogger->setSeekRequest(t0+tt); - } - this->update(); -} - -void Window::paintEvent(QPaintEvent *event) { - if (events.size() == 0) return; - - QElapsedTimer timer; - timer.start(); - - uint64_t t0 = events.begin().key(); - - //p.drawRect(0, 0, 600, 100); - - // TODO: we really don't have to redraw this every time, only on updates to events - float vEgo = 0.; - int this_event_size = events.size(); - if (last_event_size != this_event_size) { - if (px != NULL) delete px; - px = new QPixmap(1920, 600); - px->fill(QColor(0xd8, 0xd8, 0xd8)); - - QPainter tt(px); - tt.setBrush(Qt::cyan); - - int lt = -1; - int lvv = 0; - for (auto e : events) { - auto type = e.which(); - //printf("%lld %d\n", e.getLogMonoTime()-t0, type); - if (type == cereal::Event::CAR_STATE) { - vEgo = e.getCarState().getVEgo(); - } else if (type == cereal::Event::CONTROLS_STATE) { - auto controlsState = e.getControlsState(); - uint64_t t = (e.getLogMonoTime()-t0); - int enabled = controlsState.getState() == cereal::ControlsState::OpenpilotState::ENABLED; - int rt = timeToPixel(t); // 250 ms per pixel - if (rt != lt) { - int vv = vEgo*8.0; - if (lt != -1) { - tt.setPen(Qt::red); - tt.drawLine(lt, 300-lvv, rt, 300-vv); - - if (enabled) { - tt.setPen(Qt::green); - } else { - tt.setPen(Qt::blue); - } - - tt.drawLine(rt, 300, rt, 600); - } - lt = rt; - lvv = vv; - } - } - } - tt.end(); - last_event_size = this_event_size; - if (lrs.find(seg_add) != lrs.end() && lrs[seg_add]->is_done) { - while (!addSegment(++seg_add)); - } - } - - QPainter p(this); - if (px != NULL) p.drawPixmap(0, 0, 1920, 600, *px); - - p.setBrush(Qt::cyan); - - uint64_t ct = unlogger->getCurrentTime(); - if (ct != 0) { - addSegment((((ct-t0)*1e-9)/60)+1); - int rrt = timeToPixel(ct-t0); - p.drawRect(rrt-1, 0, 2, 600); - - timeLE->setText(QString("%1").arg((ct-t0)*1e-9, '8', 'f', 2)); - } - - p.end(); - - if (timer.elapsed() > 50) { - qDebug() << "paint in" << timer.elapsed() << "ms"; - } -} - -int main(int argc, char *argv[]) { - QApplication app(argc, argv); - - QString route(argv[1]); - - int use_api = QString::compare(QString("use_api"), route, Qt::CaseInsensitive) == 0; - int seek = QString(argv[2]).toInt(); - printf("seek: %d\n", seek); - route = route.replace("|", "/"); - if (route == "") { - printf("usage %s: \n", argv[0]); - exit(0); - //route = "3a5d6ac1c23e5536/2019-10-29--10-06-58"; - //route = "0006c839f32a6f99/2019-02-18--06-21-29"; - //route = "02ec6bea180a4d36/2019-10-25--10-18-09"; - } - - Window window(route, seek, use_api); - - window.resize(1920, 800); - window.setWindowTitle("nui unlogger"); - window.show(); - - return app.exec(); -} - diff --git a/tools/nui/nui b/tools/nui/nui deleted file mode 100755 index e784247b4..000000000 --- a/tools/nui/nui +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -if [ $# -gt 0 ]; then - if [ "$INTERNAL" = 1 ]; then - ./_nui "$1" - else - ./get_files_comma_api.py $1 - if [ -f ./_nui ]; then - ./_nui use_api - elif [ -f _nui.app/Contents/MacOS/_nui ]; then - ./_nui.app/Contents/MacOS/_nui use_api - else - echo "nui not found, please build it" - fi - fi -else - echo "Please Enter a Route" -fi diff --git a/tools/nui/test/.gitignore b/tools/nui/test/.gitignore deleted file mode 100644 index 9daeafb98..000000000 --- a/tools/nui/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -test diff --git a/tools/nui/test/TestFrameReader.cpp b/tools/nui/test/TestFrameReader.cpp deleted file mode 100644 index 55327ac31..000000000 --- a/tools/nui/test/TestFrameReader.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "../../clib/FrameReader.hpp" -#include "TestFrameReader.hpp" - -void TestFrameReader::frameread() { - QElapsedTimer t; - t.start(); - FrameReader fr("3a5d6ac1c23e5536/2019-10-29--10-06-58/2/fcamera.hevc"); - fr.get(2); - //QThread::sleep(10); - qDebug() << t.nsecsElapsed()*1e-9 << "seconds"; -} - -QTEST_MAIN(TestFrameReader) - diff --git a/tools/nui/test/TestFrameReader.hpp b/tools/nui/test/TestFrameReader.hpp deleted file mode 100644 index 59c07a3eb..000000000 --- a/tools/nui/test/TestFrameReader.hpp +++ /dev/null @@ -1,8 +0,0 @@ -#include - -class TestFrameReader : public QObject { -Q_OBJECT -private slots: - void frameread(); -}; - diff --git a/tools/nui/test/test.pro b/tools/nui/test/test.pro deleted file mode 100644 index 2e345f66d..000000000 --- a/tools/nui/test/test.pro +++ /dev/null @@ -1,16 +0,0 @@ -###################################################################### -# Automatically generated by qmake (3.0) Thu Oct 31 16:05:48 2019 -###################################################################### - -QT += testlib -TEMPLATE = app -TARGET = test -INCLUDEPATH += . ../ - -# Input -SOURCES += TestFrameReader.cpp ../../clib/FrameReader.cpp -HEADERS = TestFrameReader.hpp ../../clib/FrameReader.hpp - -CONFIG += c++14 - -LIBS += -lavformat -lavcodec -lavutil -lswscale