#include #include #include #include #include "cereal/messaging/messaging.h" #include "cereal/visionipc/visionipc_client.h" #include "selfdrive/common/clutil.h" #include "selfdrive/common/params.h" #include "selfdrive/common/swaglog.h" #include "selfdrive/common/util.h" #include "selfdrive/hardware/hw.h" #include "selfdrive/modeld/models/driving.h" ExitHandler do_exit; // globals bool live_calib_seen; mat3 cur_transform; std::mutex transform_lock; void calibration_thread(bool wide_camera) { set_thread_name("calibration"); set_realtime_priority(50); SubMaster sm({"liveCalibration"}); /* import numpy as np from common.transformations.model import medmodel_frame_from_road_frame medmodel_frame_from_ground = medmodel_frame_from_road_frame[:, (0, 1, 3)] ground_from_medmodel_frame = np.linalg.inv(medmodel_frame_from_ground) */ Eigen::Matrix ground_from_medmodel_frame; ground_from_medmodel_frame << 0.00000000e+00, 0.00000000e+00, 1.00000000e+00, -1.09890110e-03, 0.00000000e+00, 2.81318681e-01, -1.84808520e-20, 9.00738606e-04,-4.28751576e-02; Eigen::Matrix cam_intrinsics = Eigen::Matrix(wide_camera ? ecam_intrinsic_matrix.v : fcam_intrinsic_matrix.v); const mat3 yuv_transform = get_model_yuv_transform(); while (!do_exit) { sm.update(100); if(sm.updated("liveCalibration")){ auto extrinsic_matrix = sm["liveCalibration"].getLiveCalibration().getExtrinsicMatrix(); Eigen::Matrix extrinsic_matrix_eigen; for (int i = 0; i < 4*3; i++){ extrinsic_matrix_eigen(i / 4, i % 4) = extrinsic_matrix[i]; } auto camera_frame_from_road_frame = cam_intrinsics * extrinsic_matrix_eigen; Eigen::Matrix camera_frame_from_ground; camera_frame_from_ground.col(0) = camera_frame_from_road_frame.col(0); camera_frame_from_ground.col(1) = camera_frame_from_road_frame.col(1); camera_frame_from_ground.col(2) = camera_frame_from_road_frame.col(3); auto warp_matrix = camera_frame_from_ground * ground_from_medmodel_frame; mat3 transform = {}; for (int i=0; i<3*3; i++) { transform.v[i] = warp_matrix(i / 3, i % 3); } mat3 model_transform = matmul3(yuv_transform, transform); std::lock_guard lk(transform_lock); cur_transform = model_transform; live_calib_seen = true; } } } void run_model(ModelState &model, VisionIpcClient &vipc_client) { // messaging PubMaster pm({"modelV2", "cameraOdometry"}); SubMaster sm({"lateralPlan", "roadCameraState"}); // setup filter to track dropped frames FirstOrderFilter frame_dropped_filter(0., 10., 1. / MODEL_FREQ); uint32_t frame_id = 0, last_vipc_frame_id = 0; double last = 0; int desire = -1; uint32_t run_count = 0; while (!do_exit) { VisionIpcBufExtra extra = {}; VisionBuf *buf = vipc_client.recv(&extra); if (buf == nullptr) continue; transform_lock.lock(); mat3 model_transform = cur_transform; const bool run_model_this_iter = live_calib_seen; transform_lock.unlock(); // TODO: path planner timeout? sm.update(0); desire = ((int)sm["lateralPlan"].getLateralPlan().getDesire()); frame_id = sm["roadCameraState"].getRoadCameraState().getFrameId(); if (run_model_this_iter) { run_count++; float vec_desire[DESIRE_LEN] = {0}; if (desire >= 0 && desire < DESIRE_LEN) { vec_desire[desire] = 1.0; } double mt1 = millis_since_boot(); ModelDataRaw model_buf = model_eval_frame(&model, buf->buf_cl, buf->width, buf->height, model_transform, vec_desire); double mt2 = millis_since_boot(); float model_execution_time = (mt2 - mt1) / 1000.0; // tracked dropped frames uint32_t vipc_dropped_frames = extra.frame_id - last_vipc_frame_id - 1; float frames_dropped = frame_dropped_filter.update((float)std::min(vipc_dropped_frames, 10U)); if (run_count < 10) { // let frame drops warm up frame_dropped_filter.reset(0); frames_dropped = 0.; } float frame_drop_ratio = frames_dropped / (1 + frames_dropped); model_publish(pm, extra.frame_id, frame_id, frame_drop_ratio, model_buf, extra.timestamp_eof, model_execution_time, kj::ArrayPtr(model.output.data(), model.output.size())); posenet_publish(pm, extra.frame_id, vipc_dropped_frames, model_buf, extra.timestamp_eof); //printf("model process: %.2fms, from last %.2fms, vipc_frame_id %u, frame_id, %u, frame_drop %.3f\n", mt2 - mt1, mt1 - last, extra.frame_id, frame_id, frame_drop_ratio); last = mt1; last_vipc_frame_id = extra.frame_id; } } } int main(int argc, char **argv) { set_realtime_priority(54); if (Hardware::EON()) { set_core_affinity(2); } else if (Hardware::TICI()) { set_core_affinity(7); } bool wide_camera = Hardware::TICI() ? Params().getBool("EnableWideCamera") : false; // start calibration thread std::thread thread = std::thread(calibration_thread, wide_camera); // cl init cl_device_id device_id = cl_get_device_id(CL_DEVICE_TYPE_DEFAULT); cl_context context = CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err)); // init the models ModelState model; model_init(&model, device_id, context); LOGW("models loaded, modeld starting"); VisionIpcClient vipc_client = VisionIpcClient("camerad", wide_camera ? VISION_STREAM_YUV_WIDE : VISION_STREAM_YUV_BACK, true, device_id, context); while (!do_exit && !vipc_client.connect(false)) { util::sleep_for(100); } // run the models // vipc_client.connected is false only when do_exit is true if (vipc_client.connected) { const VisionBuf *b = &vipc_client.buffers[0]; LOGW("connected with buffer size: %d (%d x %d)", b->len, b->width, b->height); run_model(model, vipc_client); } model_free(&model); LOG("joining calibration thread"); thread.join(); CL_CHECK(clReleaseContext(context)); return 0; }