#include "selfdrive/modeld/models/driving.h" #include #include #include #include #include #include "selfdrive/common/clutil.h" #include "selfdrive/common/params.h" #include "selfdrive/common/timing.h" constexpr float FCW_THRESHOLD_5MS2_HIGH = 0.15; constexpr float FCW_THRESHOLD_5MS2_LOW = 0.05; constexpr float FCW_THRESHOLD_3MS2 = 0.7; std::array prev_brake_5ms2_probs = {0,0,0,0,0}; std::array prev_brake_3ms2_probs = {0,0,0}; // #define DUMP_YUV template constexpr const kj::ArrayPtr to_kj_array_ptr(const std::array &arr) { return kj::ArrayPtr(arr.data(), arr.size()); } void model_init(ModelState* s, cl_device_id device_id, cl_context context) { s->frame = new ModelFrame(device_id, context); s->wide_frame = new ModelFrame(device_id, context); #ifdef USE_THNEED s->m = std::make_unique("../../models/supercombo.thneed", #elif USE_ONNX_MODEL s->m = std::make_unique("../../models/supercombo.onnx", #else s->m = std::make_unique("../../models/supercombo.dlc", #endif &s->output[0], NET_OUTPUT_SIZE, USE_GPU_RUNTIME, true); #ifdef TEMPORAL s->m->addRecurrent(&s->output[OUTPUT_SIZE], TEMPORAL_SIZE); #endif #ifdef DESIRE s->m->addDesire(s->pulse_desire, DESIRE_LEN); #endif #ifdef TRAFFIC_CONVENTION const int idx = Params().getBool("IsRHD") ? 1 : 0; s->traffic_convention[idx] = 1.0; s->m->addTrafficConvention(s->traffic_convention, TRAFFIC_CONVENTION_LEN); #endif } ModelOutput* model_eval_frame(ModelState* s, VisionBuf* buf, VisionBuf* wbuf, const mat3 &transform, const mat3 &transform_wide, float *desire_in) { #ifdef DESIRE if (desire_in != NULL) { for (int i = 1; i < DESIRE_LEN; i++) { // Model decides when action is completed // so desire input is just a pulse triggered on rising edge if (desire_in[i] - s->prev_desire[i] > .99) { s->pulse_desire[i] = desire_in[i]; } else { s->pulse_desire[i] = 0.0; } s->prev_desire[i] = desire_in[i]; } } #endif // if getInputBuf is not NULL, net_input_buf will be auto net_input_buf = s->frame->prepare(buf->buf_cl, buf->width, buf->height, transform, static_cast(s->m->getInputBuf())); s->m->addImage(net_input_buf, s->frame->buf_size); if (wbuf != nullptr) { auto net_extra_buf = s->wide_frame->prepare(wbuf->buf_cl, wbuf->width, wbuf->height, transform_wide, static_cast(s->m->getExtraBuf())); s->m->addExtra(net_extra_buf, s->wide_frame->buf_size); } s->m->execute(); return (ModelOutput*)&s->output; } void model_free(ModelState* s) { delete s->frame; } void fill_lead(cereal::ModelDataV2::LeadDataV3::Builder lead, const ModelOutputLeads &leads, int t_idx, float prob_t) { std::array lead_t = {0.0, 2.0, 4.0, 6.0, 8.0, 10.0}; const auto &best_prediction = leads.get_best_prediction(t_idx); lead.setProb(sigmoid(leads.prob[t_idx])); lead.setProbTime(prob_t); std::array lead_x, lead_y, lead_v, lead_a; std::array lead_x_std, lead_y_std, lead_v_std, lead_a_std; for (int i=0; i desire_state_softmax; softmax(meta_data.desire_state_prob.array.data(), desire_state_softmax.data(), DESIRE_LEN); std::array desire_pred_softmax; for (int i=0; i lat_long_t = {2,4,6,8,10}; std::array gas_disengage_sigmoid, brake_disengage_sigmoid, steer_override_sigmoid, brake_3ms2_sigmoid, brake_4ms2_sigmoid, brake_5ms2_sigmoid; for (int i=0; i threshold; } for (int i=0; i FCW_THRESHOLD_3MS2; } auto disengage = meta.initDisengagePredictions(); disengage.setT(to_kj_array_ptr(lat_long_t)); disengage.setGasDisengageProbs(to_kj_array_ptr(gas_disengage_sigmoid)); disengage.setBrakeDisengageProbs(to_kj_array_ptr(brake_disengage_sigmoid)); disengage.setSteerOverrideProbs(to_kj_array_ptr(steer_override_sigmoid)); disengage.setBrake3MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_3ms2_sigmoid)); disengage.setBrake4MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_4ms2_sigmoid)); disengage.setBrake5MetersPerSecondSquaredProbs(to_kj_array_ptr(brake_5ms2_sigmoid)); meta.setEngagedProb(sigmoid(meta_data.engaged_prob)); meta.setDesirePrediction(to_kj_array_ptr(desire_pred_softmax)); meta.setDesireState(to_kj_array_ptr(desire_state_softmax)); meta.setHardBrakePredicted(above_fcw_threshold); } template void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z) { xyzt.setT(to_kj_array_ptr(t)); xyzt.setX(to_kj_array_ptr(x)); xyzt.setY(to_kj_array_ptr(y)); xyzt.setZ(to_kj_array_ptr(z)); } template void fill_xyzt(cereal::ModelDataV2::XYZTData::Builder xyzt, const std::array &t, const std::array &x, const std::array &y, const std::array &z, const std::array &x_std, const std::array &y_std, const std::array &z_std) { fill_xyzt(xyzt, t, x, y, z); xyzt.setXStd(to_kj_array_ptr(x_std)); xyzt.setYStd(to_kj_array_ptr(y_std)); xyzt.setZStd(to_kj_array_ptr(z_std)); } void fill_plan(cereal::ModelDataV2::Builder &framed, const ModelOutputPlanPrediction &plan) { std::array pos_x, pos_y, pos_z; std::array pos_x_std, pos_y_std, pos_z_std; std::array vel_x, vel_y, vel_z; std::array rot_x, rot_y, rot_z; std::array rot_rate_x, rot_rate_y, rot_rate_z; for(int i=0; i &plan_t, const ModelOutputLaneLines &lanes) { const auto &left_far = lanes.get_lane_idx(0); const auto &left_near = lanes.get_lane_idx(1); const auto &right_near = lanes.get_lane_idx(2); const auto &right_far = lanes.get_lane_idx(3); std::array left_far_y, left_far_z; std::array left_near_y, left_near_z; std::array right_near_y, right_near_z; std::array right_far_y, right_far_z; for (int j=0; j &plan_t, const ModelOutputRoadEdges &edges) { std::array left_y, left_z; std::array right_y, right_z; for (int j=0; j plan_t; std::fill_n(plan_t.data(), plan_t.size(), NAN); plan_t[0] = 0.0; for (int xidx=1, tidx=0; xidx t_offsets = {0.0, 2.0, 4.0}; for (int i=0; i raw_pred, const bool valid) { const uint32_t frame_age = (frame_id > vipc_frame_id) ? (frame_id - vipc_frame_id) : 0; MessageBuilder msg; auto framed = msg.initEvent(valid).initModelV2(); framed.setFrameId(vipc_frame_id); framed.setFrameIdExtra(vipc_frame_id_extra); framed.setFrameAge(frame_age); framed.setFrameDropPerc(frame_drop * 100); framed.setTimestampEof(timestamp_eof); framed.setModelExecutionTime(model_execution_time); if (send_raw_pred) { framed.setRawPredictions(raw_pred.asBytes()); } fill_model(framed, net_outputs); pm.send("modelV2", msg); } void posenet_publish(PubMaster &pm, uint32_t vipc_frame_id, uint32_t vipc_dropped_frames, const ModelOutput &net_outputs, uint64_t timestamp_eof, const bool valid) { MessageBuilder msg; const auto &v_mean = net_outputs.pose.velocity_mean; const auto &r_mean = net_outputs.pose.rotation_mean; const auto &v_std = net_outputs.pose.velocity_std; const auto &r_std = net_outputs.pose.rotation_std; auto posenetd = msg.initEvent(valid && (vipc_dropped_frames < 1)).initCameraOdometry(); posenetd.setTrans({v_mean.x, v_mean.y, v_mean.z}); posenetd.setRot({r_mean.x, r_mean.y, r_mean.z}); posenetd.setTransStd({exp(v_std.x), exp(v_std.y), exp(v_std.z)}); posenetd.setRotStd({exp(r_std.x), exp(r_std.y), exp(r_std.z)}); posenetd.setTimestampEof(timestamp_eof); posenetd.setFrameId(vipc_frame_id); pm.send("cameraOdometry", msg); }