message builder (#2161)

* use custom MessageBuilder from cereal

* use MesageBuilder.toBytes

* bump cereal

* unused

* another one

* fix ui msg

Co-authored-by: deanlee <deanlee3@gmail.com>
pull/2164/head
Adeeb Shihadeh 2020-09-12 21:48:15 -07:00 committed by GitHub
parent 71562d6180
commit 4d5733924a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 71 additions and 163 deletions

2
cereal

@ -1 +1 @@
Subproject commit 177bc413f78579d0be19ada7ee1fa093b79afb7c
Subproject commit ba61dbdf46a58d8e895360daa646b8797961443e

View File

@ -190,8 +190,8 @@ void can_recv(PubMaster &pm) {
uint64_t start_time = nanos_since_boot();
// create message
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
MessageBuilder msg;
auto event = msg.initEvent();
event.setLogMonoTime(start_time);
int recv = panda->can_receive(event);
@ -275,10 +275,8 @@ void can_health_thread() {
// Broadcast empty health message when panda is not yet connected
while (!panda){
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto healthData = event.initHealth();
MessageBuilder msg;
auto healthData = msg.initEvent().initHealth();
healthData.setHwType(cereal::HealthData::HwType::UNKNOWN);
pm.send("health", msg);
@ -287,10 +285,8 @@ void can_health_thread() {
// run at 2hz
while (!do_exit && panda->connected) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto healthData = event.initHealth();
MessageBuilder msg;
auto healthData = msg.initEvent().initHealth();
health_t health = panda->get_health();
@ -460,10 +456,8 @@ void hardware_control_thread() {
static void pigeon_publish_raw(PubMaster &pm, std::string dat) {
// create message
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto ublox_raw = event.initUbloxRaw(dat.length());
MessageBuilder msg;
auto ublox_raw = msg.initEvent().initUbloxRaw(dat.length());
memcpy(ublox_raw.begin(), dat.data(), dat.length());
pm.send("ubloxRaw", msg);

View File

@ -1,8 +1,7 @@
#include <vector>
#include <tuple>
#include <string>
#include "common/timing.h"
#include <capnp/serialize.h>
#include "messaging.hpp"
#include "cereal/gen/cpp/log.capnp.h"
#include "cereal/gen/cpp/car.capnp.h"
@ -16,9 +15,8 @@ typedef struct {
extern "C" {
void can_list_to_can_capnp_cpp(const std::vector<can_frame> &can_list, std::string &out, bool sendCan, bool valid) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
MessageBuilder msg;
auto event = msg.initEvent();
event.setValid(valid);
auto canData = sendCan ? event.initSendcan(can_list.size()) : event.initCan(can_list.size());
@ -29,8 +27,7 @@ void can_list_to_can_capnp_cpp(const std::vector<can_frame> &can_list, std::stri
canData[j].setDat(kj::arrayPtr((uint8_t*)it->dat.data(), it->dat.size()));
canData[j].setSrc(it->src);
}
auto words = capnp::messageToFlatArray(msg);
auto bytes = words.asBytes();
auto bytes = msg.toBytes();
out.append((const char *)bytes.begin(), bytes.size());
}

View File

@ -343,11 +343,8 @@ void* frontview_thread(void *arg) {
// send frame event
{
if (s->pm != NULL) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto framed = event.initFrontFrame();
MessageBuilder msg;
auto framed = msg.initEvent().initFrontFrame();
framed.setFrameId(frame_data.frame_id);
framed.setEncodeId(cnt);
framed.setTimestampEof(frame_data.timestamp_eof);
@ -474,11 +471,8 @@ void* wideview_thread(void *arg) {
// send frame event
{
if (s->pm != NULL) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto framed = event.initWideFrame();
MessageBuilder msg;
auto framed = msg.initEvent().initWideFrame();
framed.setFrameId(frame_data.frame_id);
framed.setEncodeId(cnt);
framed.setTimestampEof(frame_data.timestamp_eof);
@ -740,11 +734,8 @@ void* processing_thread(void *arg) {
// send frame event
{
if (s->pm != NULL) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto framed = event.initFrame();
MessageBuilder msg;
auto framed = msg.initEvent().initFrame();
framed.setFrameId(frame_data.frame_id);
framed.setEncodeId(cnt);
framed.setTimestampEof(frame_data.timestamp_eof);
@ -828,11 +819,8 @@ void* processing_thread(void *arg) {
free(row);
jpeg_finish_compress(&cinfo);
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto thumbnaild = event.initThumbnail();
MessageBuilder msg;
auto thumbnaild = msg.initEvent().initThumbnail();
thumbnaild.setFrameId(frame_data.frame_id);
thumbnaild.setTimestampEof(frame_data.timestamp_eof);
thumbnaild.setThumbnail(kj::arrayPtr((const uint8_t*)thumbnail_buffer, thumbnail_len));

View File

@ -62,10 +62,8 @@ int main() {
uint64_t modem_uptime_v = arm_cntpct() / 19200ULL; // 19.2 mhz clock
#endif
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(boottime);
auto clocks = event.initClocks();
MessageBuilder msg;
auto clocks = msg.initEvent().initClocks();
clocks.setBootTimeNanos(boottime);
clocks.setMonotonicNanos(monotonic);

View File

@ -193,10 +193,8 @@ inline bool UbloxMsgParser::valid_so_far() {
kj::Array<capnp::word> UbloxMsgParser::gen_solution() {
nav_pvt_msg *msg = (nav_pvt_msg *)&msg_parse_buf[UBLOX_HEADER_SIZE];
capnp::MallocMessageBuilder msg_builder;
cereal::Event::Builder event = msg_builder.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto gpsLoc = event.initGpsLocationExternal();
MessageBuilder msg_builder;
auto gpsLoc = msg_builder.initEvent().initGpsLocationExternal();
gpsLoc.setSource(cereal::GpsLocationData::SensorSource::UBLOX);
gpsLoc.setFlags(msg->flags);
gpsLoc.setLatitude(msg->lat * 1e-07);
@ -236,11 +234,8 @@ kj::Array<capnp::word> UbloxMsgParser::gen_raw() {
return kj::Array<capnp::word>();
}
rxm_raw_msg_extra *measurements = (rxm_raw_msg_extra *)&msg_parse_buf[UBLOX_HEADER_SIZE + sizeof(rxm_raw_msg)];
capnp::MallocMessageBuilder msg_builder;
cereal::Event::Builder event = msg_builder.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto gnss = event.initUbloxGnss();
auto mr = gnss.initMeasurementReport();
MessageBuilder msg_builder;
auto mr = msg_builder.initEvent().initUbloxGnss().initMeasurementReport();
mr.setRcvTow(msg->rcvTow);
mr.setGpsWeek(msg->week);
mr.setLeapSeconds(msg->leapS);
@ -295,11 +290,8 @@ kj::Array<capnp::word> UbloxMsgParser::gen_nav_data() {
nav_frame_buffer[msg->gnssId][msg->svid][subframeId] = words;
if(nav_frame_buffer[msg->gnssId][msg->svid].size() == 5) {
EphemerisData ephem_data(msg->svid, nav_frame_buffer[msg->gnssId][msg->svid]);
capnp::MallocMessageBuilder msg_builder;
cereal::Event::Builder event = msg_builder.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto gnss = event.initUbloxGnss();
auto eph = gnss.initEphemeris();
MessageBuilder msg_builder;
auto eph = msg_builder.initEvent().initUbloxGnss().initEphemeris();
eph.setSvId(ephem_data.svId);
eph.setToc(ephem_data.toc);
eph.setGpsWeek(ephem_data.gpsWeek);
@ -343,11 +335,8 @@ kj::Array<capnp::word> UbloxMsgParser::gen_nav_data() {
kj::Array<capnp::word> UbloxMsgParser::gen_mon_hw() {
mon_hw_msg *msg = (mon_hw_msg *)&msg_parse_buf[UBLOX_HEADER_SIZE];
capnp::MallocMessageBuilder msg_builder;
cereal::Event::Builder event = msg_builder.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto gnss = event.initUbloxGnss();
auto hwStatus = gnss.initHwStatus();
MessageBuilder msg_builder;
auto hwStatus = msg_builder.initEvent().initUbloxGnss().initHwStatus();
hwStatus.setNoisePerMS(msg->noisePerMS);
hwStatus.setAgcCnt(msg->agcCnt);
hwStatus.setAStatus((cereal::UbloxGnss::HwStatus::AntennaSupervisorState) msg->aStatus);

View File

@ -47,15 +47,11 @@ Message * poll_ubloxraw_msg(Poller * poller) {
size_t consuming = min(len - consumed, 128);
if(consumed < len) {
// create message
capnp::MallocMessageBuilder msg_builder;
cereal::Event::Builder event = msg_builder.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto ublox_raw = event.initUbloxRaw(consuming);
MessageBuilder msg_builder;
auto ublox_raw = msg_builder.initEvent().initUbloxRaw(consuming);
memcpy(ublox_raw.begin(), (void *)(data + consumed), consuming);
auto words = capnp::messageToFlatArray(msg_builder);
auto bytes = words.asBytes();
auto bytes = msg_builder.toBytes();
Message * msg = new ZMQMessage();
msg->init((char*)bytes.begin(), bytes.size());

View File

@ -40,10 +40,8 @@ int main() {
continue;
}
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto androidEntry = event.initAndroidLog();
MessageBuilder msg;
auto androidEntry = msg.initEvent().initAndroidLog();
androidEntry.setId(log_msg.id());
androidEntry.setTs(entry.tv_sec * 1000000000ULL + entry.tv_nsec);
androidEntry.setPriority(entry.priority);

View File

@ -53,12 +53,10 @@ int main(int argc, char *argv[]) {
}
}
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
MessageBuilder msg;
// Build message
auto androidEntry = event.initAndroidLog();
auto androidEntry = msg.initEvent().initAndroidLog();
androidEntry.setTs(timestamp);
androidEntry.setMessage(json11::Json(kv).dump());
if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));

View File

@ -12,22 +12,17 @@
#include <pthread.h>
#include <bzlib.h>
#include "messaging.hpp"
#include "common/swaglog.h"
#include "logger.h"
#include <capnp/serialize.h>
#include "cereal/gen/cpp/log.capnp.h"
static void log_sentinel(LoggerState *s, cereal::Sentinel::SentinelType type) {
capnp::MallocMessageBuilder msg;
auto event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto sen = event.initSentinel();
MessageBuilder msg;
auto sen = msg.initEvent().initSentinel();
sen.setType(type);
auto words = capnp::messageToFlatArray(msg);
auto bytes = words.asBytes();
auto bytes = msg.toBytes();
logger_log(s, bytes.begin(), bytes.size(), true);
}

View File

@ -300,10 +300,8 @@ void encoder_thread(bool is_streaming, bool raw_clips, int cam_idx) {
}
// publish encode index
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto eidx = event.initEncodeIdx();
MessageBuilder msg;
auto eidx = msg.initEvent().initEncodeIdx();
eidx.setFrameId(extra.frame_id);
#ifdef QCOM2
eidx.setType(cereal::EncodeIndex::Type::FULL_H_E_V_C);
@ -337,10 +335,8 @@ void encoder_thread(bool is_streaming, bool raw_clips, int cam_idx) {
}
// publish encode index
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto eidx = event.initEncodeIdx();
MessageBuilder msg;
auto eidx = msg.initEvent().initEncodeIdx();
eidx.setFrameId(extra.frame_id);
eidx.setType(cereal::EncodeIndex::Type::FULL_LOSSLESS_CLIP);
eidx.setEncodeId(cnt);

View File

@ -168,11 +168,8 @@ DMonitoringResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_
void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringResult &res){
// make msg
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto framed = event.initDriverState();
MessageBuilder msg;
auto framed = msg.initEvent().initDriverState();
framed.setFrameId(frame_id);
kj::ArrayPtr<const float> face_orientation(&res.face_orientation[0], ARRAYSIZE(res.face_orientation));

View File

@ -234,28 +234,19 @@ void fill_longi(cereal::ModelData::LongitudinalData::Builder longi, const float
void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &net_outputs, uint64_t timestamp_eof) {
// make msg
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
uint32_t frame_age = (frame_id > vipc_frame_id) ? (frame_id - vipc_frame_id) : 0;
auto framed = event.initModel();
MessageBuilder msg;
auto framed = msg.initEvent(frame_drop < MAX_FRAME_DROP).initModel();
framed.setFrameId(vipc_frame_id);
framed.setFrameAge(frame_age);
framed.setFrameDropPerc(frame_drop * 100);
framed.setTimestampEof(timestamp_eof);
auto lpath = framed.initPath();
fill_path(lpath, net_outputs.path, false, 0);
auto left_lane = framed.initLeftLane();
fill_path(left_lane, net_outputs.left_lane, true, 1.8);
auto right_lane = framed.initRightLane();
fill_path(right_lane, net_outputs.right_lane, true, -1.8);
auto longi = framed.initLongitudinal();
fill_longi(longi, net_outputs.long_x, net_outputs.long_v, net_outputs.long_a);
fill_path(framed.initPath(), net_outputs.path, false, 0);
fill_path(framed.initLeftLane(), net_outputs.left_lane, true, 1.8);
fill_path(framed.initRightLane(), net_outputs.right_lane, true, -1.8);
fill_longi(framed.initLongitudinal(), net_outputs.long_x, net_outputs.long_v, net_outputs.long_a);
// Find the distribution that corresponds to the current lead
int mdn_max_idx = 0;
@ -265,8 +256,7 @@ void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
mdn_max_idx = i;
}
}
auto lead = framed.initLead();
fill_lead(lead, net_outputs.lead, mdn_max_idx, t_offset);
fill_lead(framed.initLead(), net_outputs.lead, mdn_max_idx, t_offset);
// Find the distribution that corresponds to the lead in 2s
mdn_max_idx = 0;
t_offset = 1;
@ -275,23 +265,14 @@ void model_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
mdn_max_idx = i;
}
}
auto lead_future = framed.initLeadFuture();
fill_lead(lead_future, net_outputs.lead, mdn_max_idx, t_offset);
auto meta = framed.initMeta();
fill_meta(meta, net_outputs.meta);
event.setValid(frame_drop < MAX_FRAME_DROP);
fill_lead(framed.initLeadFuture(), net_outputs.lead, mdn_max_idx, t_offset);
fill_meta(framed.initMeta(), net_outputs.meta);
pm.send("model", msg);
}
void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
uint32_t vipc_dropped_frames, float frame_drop, const ModelDataRaw &net_outputs, uint64_t timestamp_eof) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
float trans_arr[3];
float trans_std_arr[3];
float rot_arr[3];
@ -305,7 +286,8 @@ void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
rot_std_arr[i] = M_PI * (softplus(net_outputs.pose[9 + i]) + 1e-6) / 180.0;
}
auto posenetd = event.initCameraOdometry();
MessageBuilder msg;
auto posenetd = msg.initEvent(vipc_dropped_frames < 1).initCameraOdometry();
kj::ArrayPtr<const float> trans_vs(&trans_arr[0], 3);
posenetd.setTrans(trans_vs);
kj::ArrayPtr<const float> rot_vs(&rot_arr[0], 3);
@ -315,11 +297,8 @@ void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t frame_id,
kj::ArrayPtr<const float> rot_std_vs(&rot_std_arr[0], 3);
posenetd.setRotStd(rot_std_vs);
posenetd.setTimestampEof(timestamp_eof);
posenetd.setFrameId(vipc_frame_id);
event.setValid(vipc_dropped_frames < 1);
pm.send("cameraOdometry", msg);
}

View File

@ -38,11 +38,8 @@ int main() {
while (1) {
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto procLog = event.initProcLog();
MessageBuilder msg;
auto procLog = msg.initEvent().initProcLog();
auto orphanage = msg.getOrphanage();
// stat

View File

@ -35,14 +35,10 @@ void set_do_exit(int sig) {
void nmea_callback(GpsUtcTime timestamp, const char* nmea, int length) {
uint64_t log_time = nanos_since_boot();
uint64_t log_time_wall = nanos_since_epoch();
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(log_time);
auto nmeaData = event.initGpsNMEA();
MessageBuilder msg;
auto nmeaData = msg.initEvent().initGpsNMEA();
nmeaData.setTimestamp(timestamp);
nmeaData.setLocalWallTime(log_time_wall);
nmeaData.setNmea(nmea);
@ -52,13 +48,9 @@ void nmea_callback(GpsUtcTime timestamp, const char* nmea, int length) {
void location_callback(GpsLocation* location) {
//printf("got location callback\n");
uint64_t log_time = nanos_since_boot();
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(log_time);
auto locationData = event.initGpsLocation();
MessageBuilder msg;
auto locationData = msg.initEvent().initGpsLocation();
locationData.setFlags(location->flags);
locationData.setLatitude(location->latitude);
locationData.setLongitude(location->longitude);

View File

@ -115,11 +115,8 @@ void sensor_loop() {
}
}
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto sensor_events = event.initSensorEvents(log_events);
MessageBuilder msg;
auto sensor_events = msg.initEvent().initSensorEvents(log_events);
int log_i = 0;
for (int i = 0; i < n; i++) {

View File

@ -62,8 +62,8 @@ int sensor_loop() {
std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
uint64_t log_time = nanos_since_boot();
capnp::MallocMessageBuilder msg;
cereal::Event::Builder event = msg.initRoot<cereal::Event>();
MessageBuilder msg;
auto event = msg.initEvent();
event.setLogMonoTime(log_time);
int num_events = sensors.size();

View File

@ -7,7 +7,6 @@
#include "common/utilpp.h"
#include "common/params.h"
#include "common/touch.h"
#include "common/timing.h"
#include "common/swaglog.h"
#include "ui.hpp"
@ -149,10 +148,8 @@ static void update_offroad_layout_state(UIState *s, PubMaster *pm) {
timeout--;
}
if (prev_collapsed != s->scene.uilayout_sidebarcollapsed || prev_app != s->active_app || timeout == 0) {
capnp::MallocMessageBuilder msg;
auto event = msg.initRoot<cereal::Event>();
event.setLogMonoTime(nanos_since_boot());
auto layout = event.initUiLayoutState();
MessageBuilder msg;
auto layout = msg.initEvent().initUiLayoutState();
layout.setActiveApp(s->active_app);
layout.setSidebarCollapsed(s->scene.uilayout_sidebarcollapsed);
pm->send("offroadLayout", msg);