from collections import namedtuple from typing import Any, Dict, Tuple import matplotlib import matplotlib.pyplot as plt import numpy as np import pygame # pylint: disable=import-error from common.transformations.camera import (eon_f_frame_size, eon_f_focal_length, tici_f_frame_size, tici_f_focal_length) from selfdrive.config import RADAR_TO_CAMERA from selfdrive.config import UIParams as UP from tools.lib.lazy_property import lazy_property RED = (255, 0, 0) GREEN = (0, 255, 0) BLUE = (0, 0, 255) YELLOW = (255, 255, 0) BLACK = (0, 0, 0) WHITE = (255, 255, 255) _PATH_X = np.arange(192.) _PATH_XD = np.arange(192.) _FULL_FRAME_SIZE = { } _BB_TO_FULL_FRAME = {} _FULL_FRAME_TO_BB = {} _INTRINSICS = {} cams = [(eon_f_frame_size[0], eon_f_frame_size[1], eon_f_focal_length), (tici_f_frame_size[0], tici_f_frame_size[1], tici_f_focal_length)] for width, height, focal in cams: sz = width * height _BB_SCALE = width / 640. _BB_TO_FULL_FRAME[sz] = np.asarray([ [_BB_SCALE, 0., 0.], [0., _BB_SCALE, 0.], [0., 0., 1.]]) _FULL_FRAME_TO_BB[sz] = np.linalg.inv(_BB_TO_FULL_FRAME[sz]) _FULL_FRAME_SIZE[sz] = (width, height) _INTRINSICS[sz] = np.array([ [focal, 0., width / 2.], [0., focal, height / 2.], [0., 0., 1.]]) METER_WIDTH = 20 ModelUIData = namedtuple("ModelUIData", ["cpath", "lpath", "rpath", "lead", "lead_future"]) _COLOR_CACHE : Dict[Tuple[int, int, int], Any] = {} def find_color(lidar_surface, color): if color in _COLOR_CACHE: return _COLOR_CACHE[color] tcolor = 0 ret = 255 for x in lidar_surface.get_palette(): #print tcolor, x if x[0:3] == color: ret = tcolor break tcolor += 1 _COLOR_CACHE[color] = ret return ret def warp_points(pt_s, warp_matrix): # pt_s are the source points, nxm array. pt_d = np.dot(warp_matrix[:, :-1], pt_s.T) + warp_matrix[:, -1, None] # Divide by last dimension for representation in image space. return (pt_d[:-1, :] / pt_d[-1, :]).T def to_lid_pt(y, x): px, py = -x * UP.lidar_zoom + UP.lidar_car_x, -y * UP.lidar_zoom + UP.lidar_car_y if px > 0 and py > 0 and px < UP.lidar_x and py < UP.lidar_y: return int(px), int(py) return -1, -1 def draw_path(y, x, color, img, calibration, top_down, lid_color=None): # TODO: Remove big box. uv_model_real = warp_points(np.column_stack((x, y)), calibration.car_to_model) uv_model = np.round(uv_model_real).astype(int) uv_model_dots = uv_model[np.logical_and.reduce((np.all( # pylint: disable=no-member uv_model > 0, axis=1), uv_model[:, 0] < img.shape[1] - 1, uv_model[:, 1] < img.shape[0] - 1))] for i, j in ((-1, 0), (0, -1), (0, 0), (0, 1), (1, 0)): img[uv_model_dots[:, 1] + i, uv_model_dots[:, 0] + j] = color # draw lidar path point on lidar # find color in 8 bit if lid_color is not None and top_down is not None: tcolor = find_color(top_down[0], lid_color) for i in range(len(x)): px, py = to_lid_pt(x[i], y[i]) if px != -1: top_down[1][px, py] = tcolor def draw_steer_path(speed_ms, curvature, color, img, calibration, top_down, VM, lid_color=None): path_x = np.arange(101.) path_y = np.multiply(path_x, np.tan(np.arcsin(np.clip(path_x * curvature, -0.999, 0.999)) / 2.)) draw_path(path_y, path_x, color, img, calibration, top_down, lid_color) def draw_lead_car(closest, top_down): if closest is not None: closest_y = int(round(UP.lidar_car_y - closest * UP.lidar_zoom)) if closest_y > 0: top_down[1][int(round(UP.lidar_car_x - METER_WIDTH * 2)):int( round(UP.lidar_car_x + METER_WIDTH * 2)), closest_y] = find_color( top_down[0], (255, 0, 0)) def draw_lead_on(img, closest_x_m, closest_y_m, calibration, color, sz=10, img_offset=(0, 0)): uv = warp_points(np.asarray([closest_x_m, closest_y_m]), calibration.car_to_bb)[0] u, v = int(uv[0] + img_offset[0]), int(uv[1] + img_offset[1]) if u > 0 and u < 640 and v > 0 and v < 480 - 5: img[v - 5 - sz:v - 5 + sz, u] = color img[v - 5, u - sz:u + sz] = color return u, v def init_plots(arr, name_to_arr_idx, plot_xlims, plot_ylims, plot_names, plot_colors, plot_styles, bigplots=False): color_palette = { "r": (1, 0, 0), "g": (0, 1, 0), "b": (0, 0, 1), "k": (0, 0, 0), "y": (1, 1, 0), "p": (0, 1, 1), "m": (1, 0, 1) } if bigplots: fig = plt.figure(figsize=(6.4, 7.0)) else: fig = plt.figure() fig.set_facecolor((0.2, 0.2, 0.2)) axs = [] for pn in range(len(plot_ylims)): ax = fig.add_subplot(len(plot_ylims), 1, len(axs)+1) ax.set_xlim(plot_xlims[pn][0], plot_xlims[pn][1]) ax.set_ylim(plot_ylims[pn][0], plot_ylims[pn][1]) ax.patch.set_facecolor((0.4, 0.4, 0.4)) axs.append(ax) plots, idxs, plot_select = [], [], [] for i, pl_list in enumerate(plot_names): for j, item in enumerate(pl_list): plot, = axs[i].plot(arr[:, name_to_arr_idx[item]], label=item, color=color_palette[plot_colors[i][j]], linestyle=plot_styles[i][j]) plots.append(plot) idxs.append(name_to_arr_idx[item]) plot_select.append(i) axs[i].set_title(", ".join("%s (%s)" % (nm, cl) for (nm, cl) in zip(pl_list, plot_colors[i])), fontsize=10) axs[i].tick_params(axis="x", colors="white") axs[i].tick_params(axis="y", colors="white") axs[i].title.set_color("white") if i < len(plot_ylims) - 1: axs[i].set_xticks([]) fig.canvas.draw() renderer = fig.canvas.get_renderer() if matplotlib.get_backend() == "MacOSX": fig.draw(renderer) def draw_plots(arr): for ax in axs: ax.draw_artist(ax.patch) for i in range(len(plots)): plots[i].set_ydata(arr[:, idxs[i]]) axs[plot_select[i]].draw_artist(plots[i]) if matplotlib.get_backend() == "QT4Agg": fig.canvas.update() fig.canvas.flush_events() raw_data = renderer.tostring_rgb() x, y = fig.canvas.get_width_height() # Handle 2x scaling if len(raw_data) == 4 * x * y * 3: plot_surface = pygame.image.frombuffer(raw_data, (2*x, 2*y), "RGB").convert() plot_surface = pygame.transform.scale(plot_surface, (x, y)) else: plot_surface = pygame.image.frombuffer(raw_data, fig.canvas.get_width_height(), "RGB").convert() return plot_surface return draw_plots def draw_mpc(liveMpc, top_down): mpc_color = find_color(top_down[0], (0, 255, 0)) for p in zip(liveMpc.x, liveMpc.y): px, py = to_lid_pt(*p) top_down[1][px, py] = mpc_color class CalibrationTransformsForWarpMatrix(object): def __init__(self, num_px, model_to_full_frame, K, E): self._model_to_full_frame = model_to_full_frame self._K = K self._E = E self.num_px = num_px @property def model_to_bb(self): return _FULL_FRAME_TO_BB[self.num_px].dot(self._model_to_full_frame) @lazy_property def model_to_full_frame(self): return self._model_to_full_frame @lazy_property def car_to_model(self): return np.linalg.inv(self._model_to_full_frame).dot(self._K).dot( self._E[:, [0, 1, 3]]) @lazy_property def car_to_bb(self): return _BB_TO_FULL_FRAME[self.num_px].dot(self._K).dot(self._E[:, [0, 1, 3]]) def pygame_modules_have_loaded(): return pygame.display.get_init() and pygame.font.get_init() def draw_var(y, x, var, color, img, calibration, top_down): # otherwise drawing gets stupid var = max(1e-1, min(var, 0.7)) varcolor = tuple(np.array(color)*0.5) draw_path(y - var, x, varcolor, img, calibration, top_down) draw_path(y + var, x, varcolor, img, calibration, top_down) class ModelPoly(object): def __init__(self, model_path): if len(model_path.poly) == 0: self.valid = False return self.poly = np.array(model_path.poly) self.prob = model_path.prob self.std = model_path.std self.y = np.polyval(self.poly, _PATH_XD) self.valid = True def extract_model_data(md): return ModelUIData( cpath=ModelPoly(md.path), lpath=ModelPoly(md.leftLane), rpath=ModelPoly(md.rightLane), lead=md.lead, lead_future=md.leadFuture, ) def plot_model(m, VM, v_ego, curvature, imgw, calibration, top_down, d_poly, top_down_color=216): if calibration is None or top_down is None: return for lead in [m.lead, m.lead_future]: if lead.prob < 0.5: continue lead_dist_from_radar = lead.dist - RADAR_TO_CAMERA _, py_top = to_lid_pt(lead_dist_from_radar + lead.std, lead.relY) px, py_bottom = to_lid_pt(lead_dist_from_radar - lead.std, lead.relY) top_down[1][int(round(px - 4)):int(round(px + 4)), py_top:py_bottom] = top_down_color color = (0, int(255 * m.lpath.prob), 0) for path in [m.cpath, m.lpath, m.rpath]: if path.valid: draw_path(path.y, _PATH_XD, color, imgw, calibration, top_down, YELLOW) draw_var(path.y, _PATH_XD, path.std, color, imgw, calibration, top_down) if d_poly is not None: dpath_y = np.polyval(d_poly, _PATH_X) draw_path(dpath_y, _PATH_X, RED, imgw, calibration, top_down, RED) # draw user path from curvature draw_steer_path(v_ego, curvature, BLUE, imgw, calibration, top_down, VM, BLUE) def maybe_update_radar_points(lt, lid_overlay): ar_pts = [] if lt is not None: ar_pts = {} for track in lt: ar_pts[track.trackId] = [track.dRel, track.yRel, track.vRel, track.aRel, track.oncoming, track.stationary] for ids, pt in ar_pts.items(): px, py = to_lid_pt(pt[0], pt[1]) if px != -1: if pt[-1]: color = 240 elif pt[-2]: color = 230 else: color = 255 if int(ids) == 1: lid_overlay[px - 2:px + 2, py - 10:py + 10] = 100 else: lid_overlay[px - 2:px + 2, py - 2:py + 2] = color def get_blank_lid_overlay(UP): lid_overlay = np.zeros((UP.lidar_x, UP.lidar_y), 'uint8') # Draw the car. lid_overlay[int(round(UP.lidar_car_x - UP.car_hwidth)):int( round(UP.lidar_car_x + UP.car_hwidth)), int(round(UP.lidar_car_y - UP.car_front))] = UP.car_color lid_overlay[int(round(UP.lidar_car_x - UP.car_hwidth)):int( round(UP.lidar_car_x + UP.car_hwidth)), int(round(UP.lidar_car_y + UP.car_back))] = UP.car_color lid_overlay[int(round(UP.lidar_car_x - UP.car_hwidth)), int( round(UP.lidar_car_y - UP.car_front)):int(round( UP.lidar_car_y + UP.car_back))] = UP.car_color lid_overlay[int(round(UP.lidar_car_x + UP.car_hwidth)), int( round(UP.lidar_car_y - UP.car_front)):int(round( UP.lidar_car_y + UP.car_back))] = UP.car_color return lid_overlay