fix up model tests + tools (#21071)
* unlogger: send yuv stream * fix up model test tools * fix unlogger * rename model replay * bump cereal * test in actions * no ci for nowpull/21075/head
parent
e3f6360e45
commit
cf6d133638
|
@ -244,6 +244,21 @@ jobs:
|
|||
name: process_replay_diff.txt
|
||||
path: selfdrive/test/process_replay/diff.txt
|
||||
|
||||
#model_replay:
|
||||
# name: model replay
|
||||
# runs-on: ubuntu-20.04
|
||||
# timeout-minutes: 50
|
||||
# steps:
|
||||
# - uses: actions/checkout@v2
|
||||
# with:
|
||||
# submodules: true
|
||||
# - name: Build Docker image
|
||||
# run: eval "$BUILD"
|
||||
# - name: Run replay
|
||||
# run: |
|
||||
# ${{ env.RUN }} "scons -j$(nproc) && \
|
||||
# selfdrive/test/process_replay/model_replay.py"
|
||||
|
||||
test_longitudinal:
|
||||
name: longitudinal
|
||||
runs-on: ubuntu-20.04
|
||||
|
|
|
@ -138,8 +138,8 @@ pipeline {
|
|||
stage('Replay Tests') {
|
||||
steps {
|
||||
phone_steps("eon2", [
|
||||
["build QCOM_REPLAY", "cd selfdrive/manager && QCOM_REPLAY=1 ./build.py"],
|
||||
["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"],
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
|
||||
])
|
||||
}
|
||||
}
|
||||
|
|
11
SConstruct
11
SConstruct
|
@ -37,6 +37,10 @@ AddOption('--mpc-generate',
|
|||
action='store_true',
|
||||
help='regenerates the mpc sources')
|
||||
|
||||
AddOption('--snpe',
|
||||
action='store_true',
|
||||
help='use SNPE on PC')
|
||||
|
||||
AddOption('--external-sconscript',
|
||||
action='store',
|
||||
metavar='FILE',
|
||||
|
@ -51,7 +55,6 @@ if arch == "aarch64" and TICI:
|
|||
arch = "larch64"
|
||||
|
||||
USE_WEBCAM = os.getenv("USE_WEBCAM") is not None
|
||||
QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
|
||||
|
||||
lenv = {
|
||||
"PATH": os.environ['PATH'],
|
||||
|
@ -98,10 +101,6 @@ if arch == "aarch64" or arch == "larch64":
|
|||
cflags = ["-DQCOM", "-mcpu=cortex-a57"]
|
||||
cxxflags = ["-DQCOM", "-mcpu=cortex-a57"]
|
||||
rpath = []
|
||||
|
||||
if QCOM_REPLAY:
|
||||
cflags += ["-DQCOM_REPLAY"]
|
||||
cxxflags += ["-DQCOM_REPLAY"]
|
||||
else:
|
||||
cflags = []
|
||||
cxxflags = []
|
||||
|
@ -338,7 +337,7 @@ if GetOption("clazy"):
|
|||
qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
|
||||
qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
|
||||
|
||||
Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
|
||||
Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED', 'USE_WEBCAM')
|
||||
|
||||
# cereal and messaging are shared with the system
|
||||
SConscript(['cereal/SConscript'])
|
||||
|
|
2
cereal
2
cereal
|
@ -1 +1 @@
|
|||
Subproject commit 7609d22fb2f8ba559a2f8a9e88985dffcfb06d5d
|
||||
Subproject commit d87e7e56d931413d0625fb765c3142ed6106c47b
|
|
@ -1,13 +1,10 @@
|
|||
Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM', 'QCOM_REPLAY')
|
||||
Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'USE_WEBCAM')
|
||||
|
||||
libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'zmq', 'capnp', 'kj', visionipc, gpucommon]
|
||||
|
||||
if arch == "aarch64":
|
||||
libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui']
|
||||
if QCOM_REPLAY:
|
||||
cameras = ['cameras/camera_frame_stream.cc']
|
||||
else:
|
||||
cameras = ['cameras/camera_qcom.cc']
|
||||
cameras = ['cameras/camera_qcom.cc']
|
||||
elif arch == "larch64":
|
||||
libs += ['atomic']
|
||||
cameras = ['cameras/camera_qcom2.cc']
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
#include "selfdrive/common/util.h"
|
||||
#include "selfdrive/hardware/hw.h"
|
||||
|
||||
#if defined(QCOM) && !defined(QCOM_REPLAY)
|
||||
#ifdef QCOM
|
||||
#include "selfdrive/camerad/cameras/camera_qcom.h"
|
||||
#elif QCOM2
|
||||
#include "selfdrive/camerad/cameras/camera_qcom2.h"
|
||||
|
|
|
@ -15,7 +15,7 @@
|
|||
#include "selfdrive/common/util.h"
|
||||
#include "selfdrive/hardware/hw.h"
|
||||
|
||||
#if defined(QCOM) && !defined(QCOM_REPLAY)
|
||||
#ifdef QCOM
|
||||
#include "selfdrive/camerad/cameras/camera_qcom.h"
|
||||
#elif QCOM2
|
||||
#include "selfdrive/camerad/cameras/camera_qcom2.h"
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
Import('env', 'arch', 'SHARED', 'QCOM_REPLAY')
|
||||
Import('env', 'arch', 'SHARED')
|
||||
|
||||
if SHARED:
|
||||
fxn = env.SharedLibrary
|
||||
|
|
|
@ -37,12 +37,13 @@ if arch == "aarch64" or arch == "larch64":
|
|||
else:
|
||||
libs += ['pthread']
|
||||
|
||||
# for onnx support
|
||||
common_src += ['runners/onnxmodel.cc']
|
||||
if not GetOption('snpe'):
|
||||
# for onnx support
|
||||
common_src += ['runners/onnxmodel.cc']
|
||||
|
||||
# tell runners to use onnx
|
||||
lenv['CFLAGS'].append("-DUSE_ONNX_MODEL")
|
||||
lenv['CXXFLAGS'].append("-DUSE_ONNX_MODEL")
|
||||
# tell runners to use onnx
|
||||
lenv['CFLAGS'].append("-DUSE_ONNX_MODEL")
|
||||
lenv['CXXFLAGS'].append("-DUSE_ONNX_MODEL")
|
||||
|
||||
if arch == "Darwin":
|
||||
# fix OpenCL
|
||||
|
@ -56,10 +57,9 @@ else:
|
|||
|
||||
common_model = lenv.Object(common_src)
|
||||
|
||||
|
||||
# build thneed model
|
||||
if arch == "aarch64" or arch == "larch64":
|
||||
compiler = lenv.Program('thneed/compile', ["thneed/compile.cc" ]+common_model, LIBS=libs)
|
||||
compiler = lenv.Program('thneed/compile', ["thneed/compile.cc"]+common_model, LIBS=libs)
|
||||
cmd = f"cd {Dir('.').abspath} && {compiler[0].abspath} ../../models/supercombo.dlc ../../models/supercombo.thneed --binary"
|
||||
|
||||
lib_paths = ':'.join([Dir(p).abspath for p in lenv["LIBPATH"]])
|
||||
|
|
|
@ -1,12 +1,15 @@
|
|||
#!/bin/sh
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
|
||||
if [ -d /system ]; then
|
||||
if [ -f /TICI ]; then # QCOM2
|
||||
export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/phonelibs/snpe/larch64:$LD_LIBRARY_PATH"
|
||||
else # QCOM
|
||||
export LD_LIBRARY_PATH="/data/pythonpath/phonelibs/snpe/aarch64/:$LD_LIBRARY_PATH"
|
||||
fi
|
||||
if [ -f /TICI ]; then # QCOM2
|
||||
export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/phonelibs/snpe/larch64:$LD_LIBRARY_PATH"
|
||||
else # QCOM
|
||||
export LD_LIBRARY_PATH="/data/pythonpath/phonelibs/snpe/aarch64/:$LD_LIBRARY_PATH"
|
||||
fi
|
||||
else
|
||||
# PC
|
||||
export LD_LIBRARY_PATH="$HOME/openpilot/phonelibs/snpe/x86_64-linux-clang:/openpilot/phonelibs/snpe/x86_64:$HOME/openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH"
|
||||
# PC
|
||||
export LD_LIBRARY_PATH="$DIR/../../phonelibs/snpe/x86_64-linux-clang:$DIR/../..//openpilot/phonelibs/snpe/x86_64:$LD_LIBRARY_PATH"
|
||||
fi
|
||||
exec ./_modeld
|
||||
|
|
|
@ -12,8 +12,8 @@ fi
|
|||
docker run \
|
||||
-it \
|
||||
--rm \
|
||||
--volume $OP_ROOT:/tmp/openpilot \
|
||||
--workdir /tmp/openpilot \
|
||||
--env PYTHONPATH=/tmp/openpilot \
|
||||
--volume $OP_ROOT:$OP_ROOT \
|
||||
--workdir $PWD \
|
||||
--env PYTHONPATH=$OP_ROOT \
|
||||
ghcr.io/commaai/openpilot-base:latest \
|
||||
/bin/bash
|
||||
|
|
|
@ -5,7 +5,7 @@ import numpy as np
|
|||
from tools.lib.logreader import LogReader
|
||||
from tools.lib.framereader import FrameReader
|
||||
from tools.lib.cache import cache_path_for_file_path
|
||||
from selfdrive.test.process_replay.camera_replay import camera_replay
|
||||
from selfdrive.test.process_replay.model_replay import model_replay
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -15,7 +15,7 @@ if __name__ == "__main__":
|
|||
calib = np.load(os.path.expanduser('~/calib.npy'))
|
||||
|
||||
try:
|
||||
msgs = camera_replay(list(lr), fr, desire=desire, calib=calib)
|
||||
msgs = model_replay(list(lr), fr, desire=desire, calib=calib)
|
||||
finally:
|
||||
cache_path = cache_path_for_file_path(os.path.expanduser('~/fcamera.hevc'))
|
||||
if os.path.isfile(cache_path):
|
|
@ -8,9 +8,11 @@ from tqdm import tqdm
|
|||
|
||||
import cereal.messaging as messaging
|
||||
from cereal import log
|
||||
from cereal.visionipc.visionipc_pyx import VisionIpcServer, VisionStreamType # pylint: disable=no-name-in-module, import-error
|
||||
from common.spinner import Spinner
|
||||
from common.timeout import Timeout
|
||||
from common.transformations.camera import get_view_frame_from_road_frame
|
||||
from common.transformations.camera import get_view_frame_from_road_frame, eon_f_frame_size, tici_f_frame_size
|
||||
from selfdrive.hardware import PC
|
||||
from selfdrive.manager.process_config import managed_processes
|
||||
from selfdrive.test.openpilotci import BASE_URL, get_url
|
||||
from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
|
||||
|
@ -29,25 +31,20 @@ def replace_calib(msg, calib):
|
|||
return msg
|
||||
|
||||
|
||||
def camera_replay(lr, fr, desire=None, calib=None):
|
||||
def model_replay(lr, fr, desire=None, calib=None):
|
||||
|
||||
spinner = Spinner()
|
||||
spinner.update("starting model replay")
|
||||
|
||||
vipc_server = None
|
||||
pm = messaging.PubMaster(['roadCameraState', 'liveCalibration', 'lateralPlan'])
|
||||
sm = messaging.SubMaster(['modelV2'])
|
||||
|
||||
# TODO: add dmonitoringmodeld
|
||||
print("preparing procs")
|
||||
managed_processes['camerad'].prepare()
|
||||
managed_processes['modeld'].prepare()
|
||||
try:
|
||||
print("starting procs")
|
||||
managed_processes['camerad'].start()
|
||||
managed_processes['modeld'].start()
|
||||
time.sleep(5)
|
||||
sm.update(1000)
|
||||
print("procs started")
|
||||
|
||||
desires_by_index = {v:k for k,v in log.LateralPlan.Desire.schema.enumerants.items()}
|
||||
|
||||
|
@ -68,26 +65,33 @@ def camera_replay(lr, fr, desire=None, calib=None):
|
|||
pm.send('lateralPlan', dat)
|
||||
|
||||
f = msg.as_builder()
|
||||
img = fr.get(frame_idx, pix_fmt="rgb24")[0][:,:,::-1]
|
||||
f.roadCameraState.image = img.flatten().tobytes()
|
||||
frame_idx += 1
|
||||
|
||||
pm.send(msg.which(), f)
|
||||
|
||||
img = fr.get(frame_idx, pix_fmt="yuv420p")[0]
|
||||
if vipc_server is None:
|
||||
w, h = {int(3*w*h/2): (w, h) for (w, h) in [tici_f_frame_size, eon_f_frame_size]}[len(img)]
|
||||
vipc_server = VisionIpcServer("camerad")
|
||||
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h)
|
||||
vipc_server.start_listener()
|
||||
time.sleep(1) # wait for modeld to connect
|
||||
|
||||
vipc_server.send(VisionStreamType.VISION_STREAM_YUV_BACK, img.flatten().tobytes(), f.roadCameraState.frameId,
|
||||
f.roadCameraState.timestampSof, f.roadCameraState.timestampEof)
|
||||
|
||||
with Timeout(seconds=15):
|
||||
log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
|
||||
|
||||
spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
|
||||
|
||||
frame_idx += 1
|
||||
if frame_idx >= fr.frame_count:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
spinner.close()
|
||||
managed_processes['modeld'].stop()
|
||||
|
||||
print("replay done")
|
||||
spinner.close()
|
||||
managed_processes['modeld'].stop()
|
||||
time.sleep(2)
|
||||
managed_processes['camerad'].stop()
|
||||
return log_msgs
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -100,7 +104,7 @@ if __name__ == "__main__":
|
|||
lr = LogReader(get_url(TEST_ROUTE, 0))
|
||||
fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera"))
|
||||
|
||||
log_msgs = camera_replay(list(lr), fr)
|
||||
log_msgs = model_replay(list(lr), fr)
|
||||
|
||||
failed = False
|
||||
if not update:
|
||||
|
@ -111,8 +115,9 @@ if __name__ == "__main__":
|
|||
ignore = ['logMonoTime', 'valid',
|
||||
'modelV2.frameDropPerc',
|
||||
'modelV2.modelExecutionTime']
|
||||
tolerance = None if not PC else 1e-3
|
||||
results: Any = {TEST_ROUTE: {}}
|
||||
results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=ignore)
|
||||
results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, tolerance=tolerance, ignore_fields=ignore)
|
||||
diff1, diff2, failed = format_diff(results, ref_commit)
|
||||
|
||||
print(diff2)
|
|
@ -1 +1 @@
|
|||
b1447cc1b5492ab28a6fafddf25ade7fc66606bf
|
||||
8f7ed52c84e9e2e6e8f4d2165130b46f27e76b30
|
|
@ -29,7 +29,8 @@ SeekAbsoluteTime = namedtuple("SeekAbsoluteTime", ("secs",))
|
|||
SeekRelativeTime = namedtuple("SeekRelativeTime", ("secs",))
|
||||
TogglePause = namedtuple("TogglePause", ())
|
||||
StopAndQuit = namedtuple("StopAndQuit", ())
|
||||
VIPC_TYP = "vipc"
|
||||
VIPC_RGB = "rgb"
|
||||
VIPC_YUV = "yuv"
|
||||
|
||||
|
||||
class UnloggerWorker(object):
|
||||
|
@ -121,9 +122,14 @@ class UnloggerWorker(object):
|
|||
smsg.roadCameraState.image = bts
|
||||
|
||||
extra = (smsg.roadCameraState.frameId, smsg.roadCameraState.timestampSof, smsg.roadCameraState.timestampEof)
|
||||
data_socket.send_pyobj((cookie, VIPC_TYP, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
|
||||
data_socket.send_pyobj((cookie, VIPC_RGB, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
|
||||
data_socket.send(bts, copy=False)
|
||||
|
||||
img_yuv = self._frame_reader.get(frame_id, pix_fmt="yuv420p")
|
||||
if img_yuv is not None:
|
||||
data_socket.send_pyobj((cookie, VIPC_YUV, msg.logMonoTime, route_time, extra), flags=zmq.SNDMORE)
|
||||
data_socket.send(img_yuv.flatten().tobytes(), copy=False)
|
||||
|
||||
data_socket.send_pyobj((cookie, typ, msg.logMonoTime, route_time), flags=zmq.SNDMORE)
|
||||
data_socket.send(smsg.to_bytes(), copy=False)
|
||||
|
||||
|
@ -166,6 +172,7 @@ def _get_vipc_server(length):
|
|||
|
||||
vipc_server = VisionIpcServer("camerad")
|
||||
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_RGB_BACK, 4, True, w, h)
|
||||
vipc_server.create_buffers(VisionStreamType.VISION_STREAM_YUV_BACK, 40, False, w, h)
|
||||
vipc_server.start_listener()
|
||||
return vipc_server
|
||||
|
||||
|
@ -259,7 +266,7 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run
|
|||
print("at", route_time)
|
||||
printed_at = route_time
|
||||
|
||||
if typ not in send_funcs and typ != 'vipc':
|
||||
if typ not in send_funcs and typ not in [VIPC_RGB, VIPC_YUV]:
|
||||
if typ in address_mapping:
|
||||
# Remove so we don't keep printing warnings.
|
||||
address = address_mapping.pop(typ)
|
||||
|
@ -288,13 +295,15 @@ def unlogger_thread(command_address, forward_commands_address, data_address, run
|
|||
|
||||
# Send message.
|
||||
try:
|
||||
if typ == VIPC_TYP and (not no_visionipc):
|
||||
if vipc_server is None:
|
||||
vipc_server = _get_vipc_server(len(msg_bytes))
|
||||
if typ in [VIPC_RGB, VIPC_YUV]:
|
||||
if not no_visionipc:
|
||||
if vipc_server is None:
|
||||
vipc_server = _get_vipc_server(len(msg_bytes))
|
||||
|
||||
i, sof, eof = extra[0]
|
||||
vipc_server.send(VisionStreamType.VISION_STREAM_RGB_BACK, msg_bytes, i, sof, eof)
|
||||
if typ != VIPC_TYP:
|
||||
i, sof, eof = extra[0]
|
||||
stream = VisionStreamType.VISION_STREAM_RGB_BACK if typ == VIPC_RGB else VisionStreamType.VISION_STREAM_YUV_BACK
|
||||
vipc_server.send(stream, msg_bytes, i, sof, eof)
|
||||
else:
|
||||
send_funcs[typ](msg_bytes)
|
||||
except MultiplePublishersError:
|
||||
del send_funcs[typ]
|
||||
|
|
Loading…
Reference in New Issue