diff --git a/Jenkinsfile b/Jenkinsfile index d34cd7037..b7d58cfd6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,13 +10,29 @@ pipeline { COMMA_JWT = credentials('athena-test-jwt') } stages { - stage('EON Build/Test') { - steps { - lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){ - timeout(time: 30, unit: 'MINUTES') { - dir(path: 'release') { - sh 'pip install paramiko' - sh 'python remote_build.py' + stage('Device Tests') { + parallel { + stage('Build/Test') { + steps { + lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){ + timeout(time: 30, unit: 'MINUTES') { + dir(path: 'release') { + sh 'pip install paramiko' + sh 'python remote_build.py' + } + } + } + } + } + stage('Replay Tests') { + steps { + lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_name', quantity: 1){ + timeout(time: 45, unit: 'MINUTES') { + dir(path: 'selfdrive/test') { + sh 'pip install paramiko' + sh 'python phone_ci.py' + } + } } } } diff --git a/SConstruct b/SConstruct index 285a1743b..79304e29f 100644 --- a/SConstruct +++ b/SConstruct @@ -18,6 +18,7 @@ if arch == "aarch64" and not os.path.isdir("/system"): arch = "larch64" webcam = bool(ARGUMENTS.get("use_webcam", 0)) +QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None if arch == "aarch64" or arch == "larch64": lenv = { @@ -56,6 +57,10 @@ if arch == "aarch64" or arch == "larch64": cxxflags = ["-DQCOM", "-mcpu=cortex-a57"] rpath = ["/system/vendor/lib64"] + if QCOM_REPLAY: + cflags += ["-DQCOM_REPLAY"] + cxxflags += ["-DQCOM_REPLAY"] + else: lenv = { "PATH": "#external/bin:" + os.environ['PATH'], @@ -179,7 +184,7 @@ def abspath(x): # still needed for apks zmq = 'zmq' -Export('env', 'arch', 'zmq', 'SHARED', 'webcam') +Export('env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY') # cereal and messaging are shared with the system SConscript(['cereal/SConscript']) diff --git a/release/files_common b/release/files_common index 2ef07b565..b41b511dd 100644 --- a/release/files_common +++ b/release/files_common @@ -327,7 +327,6 @@ selfdrive/test/test_openpilot.py selfdrive/test/test_fingerprints.py selfdrive/test/test_car_models.py -selfdrive/test/process_replay/.gitignore selfdrive/test/process_replay/__init__.py selfdrive/test/process_replay/compare_logs.py selfdrive/test/process_replay/process_replay.py diff --git a/selfdrive/camerad/SConscript b/selfdrive/camerad/SConscript index 618df342c..eeb3bcb93 100644 --- a/selfdrive/camerad/SConscript +++ b/selfdrive/camerad/SConscript @@ -1,10 +1,13 @@ -Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'webcam') +Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'webcam', 'QCOM_REPLAY') libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'czmq', 'zmq', 'capnp', 'kj', visionipc, gpucommon] if arch == "aarch64": libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui'] - cameras = ['cameras/camera_qcom.cc'] + if QCOM_REPLAY: + cameras = ['cameras/camera_frame_stream.cc'] + else: + cameras = ['cameras/camera_qcom.cc'] elif arch == "larch64": libs += [] cameras = ['cameras/camera_qcom2.c'] diff --git a/selfdrive/camerad/cameras/camera_frame_stream.cc b/selfdrive/camerad/cameras/camera_frame_stream.cc index 37045f437..0a77eb118 100644 --- a/selfdrive/camerad/cameras/camera_frame_stream.cc +++ b/selfdrive/camerad/cameras/camera_frame_stream.cc @@ -90,11 +90,11 @@ void run_frame_stream(DualCameraState *s) { CameraInfo cameras_supported[CAMERA_ID_MAX] = { [CAMERA_ID_IMX298] = { - .frame_width = FRAME_WIDTH, - .frame_height = FRAME_HEIGHT, - .frame_stride = FRAME_WIDTH*3, - .bayer = false, - .bayer_flip = false, + .frame_width = FRAME_WIDTH, + .frame_height = FRAME_HEIGHT, + .frame_stride = FRAME_WIDTH*3, + .bayer = false, + .bayer_flip = false, }, [CAMERA_ID_OV8865] = { .frame_width = 1632, diff --git a/selfdrive/camerad/main.cc b/selfdrive/camerad/main.cc index f914989ea..d65913bb4 100644 --- a/selfdrive/camerad/main.cc +++ b/selfdrive/camerad/main.cc @@ -2,7 +2,7 @@ #include #include -#ifdef QCOM +#if defined(QCOM) && !defined(QCOM_REPLAY) #include "cameras/camera_qcom.h" #elif QCOM2 #include "cameras/camera_qcom2.h" @@ -403,7 +403,7 @@ void* processing_thread(void *arg) { visionbuf_sync(&s->rgb_bufs[rgb_idx], VISIONBUF_SYNC_FROM_DEVICE); -#ifdef QCOM +#if defined(QCOM) && !defined(QCOM_REPLAY) /*FILE *dump_rgb_file = fopen("/tmp/process_dump.rgb", "wb"); fwrite(s->rgb_bufs[rgb_idx].addr, s->rgb_bufs[rgb_idx].len, sizeof(uint8_t), dump_rgb_file); fclose(dump_rgb_file); @@ -515,7 +515,7 @@ void* processing_thread(void *arg) { framed.setLensTruePos(frame_data.lens_true_pos); framed.setGainFrac(frame_data.gain_frac); -#ifdef QCOM +#if defined(QCOM) && !defined(QCOM_REPLAY) kj::ArrayPtr focus_vals(&s->cameras.rear.focus[0], NUM_FOCUS); kj::ArrayPtr focus_confs(&s->cameras.rear.confidence[0], NUM_FOCUS); framed.setFocusVal(focus_vals); @@ -1215,7 +1215,7 @@ void party(VisionState *s) { zsock_signal(s->terminate_pub, 0); -#ifndef QCOM2 +#if !defined(QCOM2) && !defined(QCOM_REPLAY) LOG("joining frontview_thread"); err = pthread_join(frontview_thread_handle, NULL); assert(err == 0); @@ -1255,7 +1255,7 @@ int main(int argc, char *argv[]) { init_buffers(s); -#if defined(QCOM) || defined(QCOM2) +#if (defined(QCOM) && !defined(QCOM_REPLAY)) || defined(QCOM2) s->pm = new PubMaster({"frame", "frontFrame", "thumbnail"}); #endif @@ -1263,9 +1263,9 @@ int main(int argc, char *argv[]) { party(s); -#if defined(QCOM) || defined(QCOM2) - delete s->pm; -#endif + if (s->pm != NULL) { + delete s->pm; + } free_buffers(s); cl_free(s); diff --git a/selfdrive/common/SConscript b/selfdrive/common/SConscript index 18f9fdf4a..8ecf786ff 100644 --- a/selfdrive/common/SConscript +++ b/selfdrive/common/SConscript @@ -1,4 +1,4 @@ -Import('env', 'arch', 'SHARED') +Import('env', 'arch', 'SHARED', 'QCOM_REPLAY') if SHARED: fxn = env.SharedLibrary @@ -21,8 +21,11 @@ if arch == "aarch64": files += [ 'framebuffer.cc', 'touch.c', - 'visionbuf_ion.c', ] + if QCOM_REPLAY: + files += ['visionbuf_cl.c'] + else: + files += ['visionbuf_ion.c'] _gpu_libs = ['gui', 'adreno_utils'] elif arch == "larch64": defines = {"CLU_NO_CACHE": None} diff --git a/selfdrive/common/visionimg.cc b/selfdrive/common/visionimg.cc index a533acb59..35fe7095d 100644 --- a/selfdrive/common/visionimg.cc +++ b/selfdrive/common/visionimg.cc @@ -37,7 +37,7 @@ extern "C" void compute_aligned_width_and_height(int width, #endif void visionimg_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h) { -#ifdef QCOM +#if defined(QCOM) && !defined(QCOM_REPLAY) compute_aligned_width_and_height(ALIGN(width, 32), ALIGN(height, 32), 3, 0, 0, 512, aligned_w, aligned_h); #else *aligned_w = width; *aligned_h = height; diff --git a/selfdrive/test/.gitignore b/selfdrive/test/.gitignore index 96feaba0c..eb8c79a61 100644 --- a/selfdrive/test/.gitignore +++ b/selfdrive/test/.gitignore @@ -1,2 +1,8 @@ out/ docker_out/ + +process_replay/diff.txt +process_replay/model_diff.txt + +*.bz2 +*.hevc diff --git a/selfdrive/test/openpilotci.py b/selfdrive/test/openpilotci.py new file mode 100644 index 000000000..be8aefac7 --- /dev/null +++ b/selfdrive/test/openpilotci.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess + +BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" + +def get_url(route_name, segment_num, log_type="rlog"): + ext = "hevc" if log_type in ["fcamera", "dcamera"] else "bz2" + return BASE_URL + "%s/%s/%s.%s" % (route_name.replace("|", "/"), segment_num, log_type, ext) + +def upload_file(path, name): + from azure.storage.blob import BlockBlobService + sas_token = os.getenv("TOKEN", None) + if sas_token is None: + sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \ + --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n") + service = BlockBlobService(account_name="commadataci", sas_token=sas_token) + service.create_blob_from_path("openpilotci", name, path) + return "https://commadataci.blob.core.windows.net/openpilotci/" + name + +if __name__ == "__main__": + for f in sys.argv[1:]: + name = os.path.basename(f) + url = upload_file(f, name) + print(url) diff --git a/selfdrive/test/openpilotci_upload.py b/selfdrive/test/openpilotci_upload.py deleted file mode 100755 index 64391532f..000000000 --- a/selfdrive/test/openpilotci_upload.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -import os -import sys -import subprocess - - -def upload_file(path, name): - from azure.storage.blob import BlockBlobService - sas_token = os.getenv("TOKEN", None) - if sas_token is None: - cmd = "az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions \ - lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv" - sas_token = subprocess.check_output(cmd, shell=True).decode().strip("\n") - service = BlockBlobService(account_name="commadataci", sas_token=sas_token) - service.create_blob_from_path("openpilotci", name, path) - return "https://commadataci.blob.core.windows.net/openpilotci/" + name - -if __name__ == "__main__": - for f in sys.argv[1:]: - name = os.path.basename(f) - url = upload_file(f, name) - print(url) diff --git a/selfdrive/test/phone_ci.py b/selfdrive/test/phone_ci.py new file mode 100755 index 000000000..6ee2b396e --- /dev/null +++ b/selfdrive/test/phone_ci.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +import paramiko # pylint: disable=import-error +import os +import sys +import re +import time +import socket + +TEST_DIR = "/data/openpilotci" + +def run_test(name, test_func): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + key_file = open(os.path.join(os.path.dirname(__file__), "../../release/id_rsa_public")) + key = paramiko.RSAKey.from_private_key(key_file) + + print("SSH to phone {}".format(name)) + + # Try connecting for one minute + t_start = time.time() + while True: + try: + ssh.connect(hostname=name, port=8022, pkey=key, timeout=10) + except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError): + print("Connection failed") + if time.time() - t_start > 60: + raise + else: + break + time.sleep(1) + + conn = ssh.invoke_shell() + branch = os.environ['GIT_BRANCH'] + commit = os.environ.get('GIT_COMMIT', branch) + + conn.send("uname -a\n") + + conn.send(f"cd {TEST_DIR}\n") + conn.send("git reset --hard\n") + conn.send("git fetch origin\n") + conn.send("git checkout %s\n" % commit) + conn.send("git clean -xdf\n") + conn.send("git submodule update --init\n") + conn.send("git submodule foreach --recursive git reset --hard\n") + conn.send("git submodule foreach --recursive git clean -xdf\n") + conn.send("echo \"git took $SECONDS seconds\"\n") + + test_func(conn) + + conn.send('echo "RESULT:" $?\n') + conn.send("exit\n") + return conn + +def test_modeld(conn): + conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n") + +if __name__ == "__main__": + eon_name = os.environ.get('eon_name', None) + + conn = run_test(eon_name, test_modeld) + + dat = b"" + + while True: + recvd = conn.recv(4096) + if len(recvd) == 0: + break + + dat += recvd + sys.stdout.buffer.write(recvd) + sys.stdout.flush() + + returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE) + sys.exit(int(returns[0])) diff --git a/selfdrive/test/process_replay/.gitignore b/selfdrive/test/process_replay/.gitignore deleted file mode 100644 index 6d339d54f..000000000 --- a/selfdrive/test/process_replay/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.bz2 -diff.txt diff --git a/selfdrive/test/process_replay/camera_replay.py b/selfdrive/test/process_replay/camera_replay.py new file mode 100755 index 000000000..e2cb8cd9e --- /dev/null +++ b/selfdrive/test/process_replay/camera_replay.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import os +import sys +import time +from typing import Any +from tqdm import tqdm + +from common.android import ANDROID +if ANDROID: + os.environ['QCOM_REPLAY'] = "1" +import selfdrive.manager as manager + +from common.spinner import Spinner +import cereal.messaging as messaging +from tools.lib.framereader import FrameReader +from tools.lib.logreader import LogReader +from selfdrive.test.openpilotci import BASE_URL, get_url +from selfdrive.test.process_replay.compare_logs import compare_logs, save_log +from selfdrive.test.process_replay.test_processes import format_diff +from selfdrive.version import get_git_commit + +TEST_ROUTE = "5b7c365c50084530|2020-04-15--16-13-24" + +def camera_replay(lr, fr): + + spinner = Spinner() + + pm = messaging.PubMaster(['frame', 'liveCalibration']) + sm = messaging.SubMaster(['model']) + + # TODO: add dmonitoringmodeld + print("preparing procs") + manager.prepare_managed_process("camerad") + manager.prepare_managed_process("modeld") + try: + print("starting procs") + manager.start_managed_process("camerad") + manager.start_managed_process("modeld") + time.sleep(5) + print("procs started") + + cal = [msg for msg in lr if msg.which() == "liveCalibration"] + for msg in cal[:5]: + pm.send(msg.which(), msg.as_builder()) + + log_msgs = [] + frame_idx = 0 + for msg in tqdm(lr): + if msg.which() == "liveCalibrationd": + pm.send(msg.which(), msg.as_builder()) + elif msg.which() == "frame": + f = msg.as_builder() + img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1] + f.frame.image = img.flatten().tobytes() + frame_idx += 1 + + pm.send(msg.which(), f) + log_msgs.append(messaging.recv_one(sm.sock['model'])) + + spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count)) + + if frame_idx >= fr.frame_count: + break + except KeyboardInterrupt: + pass + + print("replay done") + spinner.close() + manager.kill_managed_process('modeld') + time.sleep(2) + manager.kill_managed_process('camerad') + return log_msgs + + +if __name__ == "__main__": + + update = "--update" in sys.argv + + lr = LogReader(get_url(TEST_ROUTE, 0)) + fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera")) + + log_msgs = camera_replay(list(lr), fr) + + if update: + ref_commit = get_git_commit() + log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) + save_log(log_fn, log_msgs) + with open("model_replay_ref_commit", "w") as f: + f.write(ref_commit) + else: + ref_commit = open("model_replay_ref_commit").read().strip() + log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit) + cmp_log = LogReader(BASE_URL + log_fn) + results: Any = {TEST_ROUTE: {}} + results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid']) + diff1, diff2, failed = format_diff(results, ref_commit) + + print(diff1) + with open("model_diff.txt", "w") as f: + f.write(diff2) + + sys.exit(int(failed)) + diff --git a/selfdrive/test/process_replay/model_replay_ref_commit b/selfdrive/test/process_replay/model_replay_ref_commit new file mode 100644 index 000000000..e98217b72 --- /dev/null +++ b/selfdrive/test/process_replay/model_replay_ref_commit @@ -0,0 +1 @@ +18e43945403c4022b1de72237d89736e1a8ab4c7 \ No newline at end of file diff --git a/selfdrive/test/process_replay/update_model.py b/selfdrive/test/process_replay/update_model.py index d3dab335e..31629563e 100755 --- a/selfdrive/test/process_replay/update_model.py +++ b/selfdrive/test/process_replay/update_model.py @@ -2,7 +2,7 @@ import os import sys -from selfdrive.test.openpilotci_upload import upload_file +from selfdrive.test.openpilotci import upload_file from selfdrive.test.process_replay.compare_logs import save_log from selfdrive.test.process_replay.test_processes import segments, get_segment from selfdrive.version import get_git_commit diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py index 37997dad0..b4243c56d 100755 --- a/selfdrive/test/process_replay/update_refs.py +++ b/selfdrive/test/process_replay/update_refs.py @@ -2,7 +2,7 @@ import os import sys -from selfdrive.test.openpilotci_upload import upload_file +from selfdrive.test.openpilotci import upload_file from selfdrive.test.process_replay.compare_logs import save_log from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS from selfdrive.test.process_replay.test_processes import segments, get_segment diff --git a/tools/lib/auth_config.py b/tools/lib/auth_config.py index 1e604970a..0cc3e33b5 100644 --- a/tools/lib/auth_config.py +++ b/tools/lib/auth_config.py @@ -1,11 +1,15 @@ import json import os +from common.android import ANDROID from common.file_helpers import mkdirs_exists_ok class MissingAuthConfigError(Exception): pass -CONFIG_DIR = os.path.expanduser('~/.comma') +if ANDROID: + CONFIG_DIR = "/tmp/.comma" +else: + CONFIG_DIR = os.path.expanduser('~/.comma') mkdirs_exists_ok(CONFIG_DIR) def get_token():