Model + camerad test (#1622)

* kind of works

* move that

* hack to get camerad to reliably terminate

* not sure why SIGTERM wasn't working before

* compare bytes

* clean up some hacks

* gitignore

* fix that

* WIP

* no reboot

* comparison works

* pretty print

* fix build

* run in jenkins

* python path

* space

* raise timeout

* new eon

* skip the copy

* spinner

* spin less

* update model ref commit

* reenable that

* clean up

* fix jenkinsfile

* parallel

* wrap it in a stage

* fix linter

* better progress

* lower timeout

Co-authored-by: Comma Device <device@comma.ai>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

Jenkinsfile

@@ -10,13 +10,29 @@ pipeline {
     COMMA_JWT = credentials('athena-test-jwt')
   }
   stages {
-    stage('EON Build/Test') {
-      steps {
-        lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){
-          timeout(time: 30, unit: 'MINUTES') {
-            dir(path: 'release') {
-              sh 'pip install paramiko'
-              sh 'python remote_build.py'
+    stage('Device Tests') {
+      parallel {
+        stage('Build/Test') {
+          steps {
+            lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){
+              timeout(time: 30, unit: 'MINUTES') {
+                dir(path: 'release') {
+                  sh 'pip install paramiko'
+                  sh 'python remote_build.py'
+                }
+              }
+            }
+          }
+        }
+        stage('Replay Tests') {
+          steps {
+            lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_name', quantity: 1){
+              timeout(time: 45, unit: 'MINUTES') {
+                dir(path: 'selfdrive/test') {
+                  sh 'pip install paramiko'
+                  sh 'python phone_ci.py'
+                }
+              }
             }
           }
         }

SConstruct

@@ -18,6 +18,7 @@ if arch == "aarch64" and not os.path.isdir("/system"):
   arch = "larch64"
 
 webcam = bool(ARGUMENTS.get("use_webcam", 0))
+QCOM_REPLAY = arch == "aarch64" and os.getenv("QCOM_REPLAY") is not None
 
 if arch == "aarch64" or arch == "larch64":
   lenv = {
@@ -56,6 +57,10 @@ if arch == "aarch64" or arch == "larch64":
     cxxflags = ["-DQCOM", "-mcpu=cortex-a57"]
     rpath = ["/system/vendor/lib64"]
 
+    if QCOM_REPLAY:
+      cflags += ["-DQCOM_REPLAY"]
+      cxxflags += ["-DQCOM_REPLAY"]
+
 else:
   lenv = {
     "PATH": "#external/bin:" + os.environ['PATH'],
@@ -179,7 +184,7 @@ def abspath(x):
 
 # still needed for apks
 zmq = 'zmq'
-Export('env', 'arch', 'zmq', 'SHARED', 'webcam')
+Export('env', 'arch', 'zmq', 'SHARED', 'webcam', 'QCOM_REPLAY')
 
 # cereal and messaging are shared with the system
 SConscript(['cereal/SConscript'])

release/files_common

@@ -327,7 +327,6 @@ selfdrive/test/test_openpilot.py
 selfdrive/test/test_fingerprints.py
 selfdrive/test/test_car_models.py
 
-selfdrive/test/process_replay/.gitignore
 selfdrive/test/process_replay/__init__.py
 selfdrive/test/process_replay/compare_logs.py
 selfdrive/test/process_replay/process_replay.py

selfdrive/camerad/SConscript

@@ -1,10 +1,13 @@
-Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'webcam')
+Import('env', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'webcam', 'QCOM_REPLAY')
 
 libs = ['m', 'pthread', common, 'jpeg', 'OpenCL', cereal, messaging, 'czmq', 'zmq', 'capnp', 'kj', visionipc, gpucommon]
 
 if arch == "aarch64":
   libs += ['gsl', 'CB', 'adreno_utils', 'EGL', 'GLESv3', 'cutils', 'ui']
-  cameras = ['cameras/camera_qcom.cc']
+  if QCOM_REPLAY:
+    cameras = ['cameras/camera_frame_stream.cc']
+  else:
+    cameras = ['cameras/camera_qcom.cc']
 elif arch == "larch64":
   libs += []
   cameras = ['cameras/camera_qcom2.c']

selfdrive/camerad/cameras/camera_frame_stream.cc

@@ -90,11 +90,11 @@ void run_frame_stream(DualCameraState *s) {
 
 CameraInfo cameras_supported[CAMERA_ID_MAX] = {
   [CAMERA_ID_IMX298] = {
-      .frame_width = FRAME_WIDTH,
-      .frame_height = FRAME_HEIGHT,
-      .frame_stride = FRAME_WIDTH*3,
-      .bayer = false,
-      .bayer_flip = false,
+    .frame_width = FRAME_WIDTH,
+    .frame_height = FRAME_HEIGHT,
+    .frame_stride = FRAME_WIDTH*3,
+    .bayer = false,
+    .bayer_flip = false,
   },
   [CAMERA_ID_OV8865] = {
     .frame_width = 1632,

selfdrive/camerad/main.cc

@@ -2,7 +2,7 @@
 #include <signal.h>
 #include <cassert>
 
-#ifdef QCOM
+#if defined(QCOM) && !defined(QCOM_REPLAY)
 #include "cameras/camera_qcom.h"
 #elif QCOM2
 #include "cameras/camera_qcom2.h"
@@ -403,7 +403,7 @@ void* processing_thread(void *arg) {
 
     visionbuf_sync(&s->rgb_bufs[rgb_idx], VISIONBUF_SYNC_FROM_DEVICE);
 
-#ifdef QCOM
+#if defined(QCOM) && !defined(QCOM_REPLAY)
     /*FILE *dump_rgb_file = fopen("/tmp/process_dump.rgb", "wb");
     fwrite(s->rgb_bufs[rgb_idx].addr, s->rgb_bufs[rgb_idx].len, sizeof(uint8_t), dump_rgb_file);
     fclose(dump_rgb_file);
@@ -515,7 +515,7 @@ void* processing_thread(void *arg) {
         framed.setLensTruePos(frame_data.lens_true_pos);
         framed.setGainFrac(frame_data.gain_frac);
 
-#ifdef QCOM
+#if defined(QCOM) && !defined(QCOM_REPLAY)
         kj::ArrayPtr<const int16_t> focus_vals(&s->cameras.rear.focus[0], NUM_FOCUS);
         kj::ArrayPtr<const uint8_t> focus_confs(&s->cameras.rear.confidence[0], NUM_FOCUS);
         framed.setFocusVal(focus_vals);
@@ -1215,7 +1215,7 @@ void party(VisionState *s) {
 
   zsock_signal(s->terminate_pub, 0);
 
-#ifndef QCOM2
+#if !defined(QCOM2) && !defined(QCOM_REPLAY)
   LOG("joining frontview_thread");
   err = pthread_join(frontview_thread_handle, NULL);
   assert(err == 0);
@@ -1255,7 +1255,7 @@ int main(int argc, char *argv[]) {
 
   init_buffers(s);
 
-#if defined(QCOM) || defined(QCOM2)
+#if (defined(QCOM) && !defined(QCOM_REPLAY)) || defined(QCOM2)
   s->pm = new PubMaster({"frame", "frontFrame", "thumbnail"});
 #endif
 
@@ -1263,9 +1263,9 @@ int main(int argc, char *argv[]) {
 
   party(s);
 
-#if defined(QCOM) || defined(QCOM2)
-  delete s->pm;
-#endif
+  if (s->pm != NULL) {
+    delete s->pm;
+  }
 
   free_buffers(s);
   cl_free(s);

selfdrive/common/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'arch', 'SHARED')
+Import('env', 'arch', 'SHARED', 'QCOM_REPLAY')
 
 if SHARED:
   fxn = env.SharedLibrary
@@ -21,8 +21,11 @@ if arch == "aarch64":
   files += [
     'framebuffer.cc',
     'touch.c',
-    'visionbuf_ion.c',
   ]
+  if QCOM_REPLAY:
+    files += ['visionbuf_cl.c']
+  else:
+    files += ['visionbuf_ion.c']
   _gpu_libs = ['gui', 'adreno_utils']
 elif arch == "larch64":
   defines = {"CLU_NO_CACHE": None}

selfdrive/common/visionimg.cc

@@ -37,7 +37,7 @@ extern "C" void compute_aligned_width_and_height(int width,
 #endif
 
 void visionimg_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h) {
-#ifdef QCOM
+#if defined(QCOM) && !defined(QCOM_REPLAY)
   compute_aligned_width_and_height(ALIGN(width, 32), ALIGN(height, 32), 3, 0, 0, 512, aligned_w, aligned_h);
 #else
   *aligned_w = width; *aligned_h = height;

selfdrive/test/.gitignore

@@ -1,2 +1,8 @@
 out/
 docker_out/
+
+process_replay/diff.txt
+process_replay/model_diff.txt
+
+*.bz2
+*.hevc

selfdrive/test/openpilotci.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+
+def get_url(route_name, segment_num, log_type="rlog"):
+  ext = "hevc" if log_type in ["fcamera", "dcamera"] else "bz2"
+  return BASE_URL + "%s/%s/%s.%s" % (route_name.replace("|", "/"), segment_num, log_type, ext)
+
+def upload_file(path, name):
+  from azure.storage.blob import BlockBlobService
+  sas_token = os.getenv("TOKEN", None)
+  if sas_token is None:
+    sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \
+                                         --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n")
+  service = BlockBlobService(account_name="commadataci", sas_token=sas_token)
+  service.create_blob_from_path("openpilotci", name, path)
+  return "https://commadataci.blob.core.windows.net/openpilotci/" + name
+
+if __name__ == "__main__":
+  for f in sys.argv[1:]:
+    name = os.path.basename(f)
+    url = upload_file(f, name)
+    print(url)

selfdrive/test/openpilotci_upload.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python3
-import os
-import sys
-import subprocess
-
-
-def upload_file(path, name):
-  from azure.storage.blob import BlockBlobService
-  sas_token = os.getenv("TOKEN", None)
-  if sas_token is None:
-    cmd = "az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions \
-           lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv"
-    sas_token = subprocess.check_output(cmd, shell=True).decode().strip("\n")
-  service = BlockBlobService(account_name="commadataci", sas_token=sas_token)
-  service.create_blob_from_path("openpilotci", name, path)
-  return "https://commadataci.blob.core.windows.net/openpilotci/" + name
-
-if __name__ == "__main__":
-  for f in sys.argv[1:]:
-    name = os.path.basename(f)
-    url = upload_file(f, name)
-    print(url)

selfdrive/test/phone_ci.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+import paramiko  # pylint: disable=import-error
+import os
+import sys
+import re
+import time
+import socket
+
+TEST_DIR = "/data/openpilotci"
+
+def run_test(name, test_func):
+  ssh = paramiko.SSHClient()
+  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+  key_file = open(os.path.join(os.path.dirname(__file__), "../../release/id_rsa_public"))
+  key = paramiko.RSAKey.from_private_key(key_file)
+
+  print("SSH to phone {}".format(name))
+
+  # Try connecting for one minute
+  t_start = time.time()
+  while True:
+    try:
+      ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
+    except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
+      print("Connection failed")
+      if time.time() - t_start > 60:
+        raise
+    else:
+      break
+    time.sleep(1)
+
+  conn = ssh.invoke_shell()
+  branch = os.environ['GIT_BRANCH']
+  commit = os.environ.get('GIT_COMMIT', branch)
+
+  conn.send("uname -a\n")
+
+  conn.send(f"cd {TEST_DIR}\n")
+  conn.send("git reset --hard\n")
+  conn.send("git fetch origin\n")
+  conn.send("git checkout %s\n" % commit)
+  conn.send("git clean -xdf\n")
+  conn.send("git submodule update --init\n")
+  conn.send("git submodule foreach --recursive git reset --hard\n")
+  conn.send("git submodule foreach --recursive git clean -xdf\n")
+  conn.send("echo \"git took $SECONDS seconds\"\n")
+
+  test_func(conn)
+
+  conn.send('echo "RESULT:" $?\n')
+  conn.send("exit\n")
+  return conn
+
+def test_modeld(conn):
+  conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
+
+if __name__ == "__main__":
+  eon_name = os.environ.get('eon_name', None)
+
+  conn = run_test(eon_name, test_modeld)
+
+  dat = b""
+
+  while True:
+    recvd = conn.recv(4096)
+    if len(recvd) == 0:
+      break
+
+    dat += recvd
+    sys.stdout.buffer.write(recvd)
+    sys.stdout.flush()
+
+  returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
+  sys.exit(int(returns[0]))

selfdrive/test/process_replay/.gitignore

@@ -1,2 +0,0 @@
-*.bz2
-diff.txt

selfdrive/test/process_replay/camera_replay.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import os
+import sys
+import time
+from typing import Any
+from tqdm import tqdm
+
+from common.android import ANDROID
+if ANDROID:
+  os.environ['QCOM_REPLAY'] = "1"
+import selfdrive.manager as manager
+
+from common.spinner import Spinner
+import cereal.messaging as messaging
+from tools.lib.framereader import FrameReader
+from tools.lib.logreader import LogReader
+from selfdrive.test.openpilotci import BASE_URL, get_url
+from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
+from selfdrive.test.process_replay.test_processes import format_diff
+from selfdrive.version import get_git_commit
+
+TEST_ROUTE = "5b7c365c50084530|2020-04-15--16-13-24"
+
+def camera_replay(lr, fr):
+
+  spinner = Spinner()
+
+  pm = messaging.PubMaster(['frame', 'liveCalibration'])
+  sm = messaging.SubMaster(['model'])
+
+  # TODO: add dmonitoringmodeld
+  print("preparing procs")
+  manager.prepare_managed_process("camerad")
+  manager.prepare_managed_process("modeld")
+  try:
+    print("starting procs")
+    manager.start_managed_process("camerad")
+    manager.start_managed_process("modeld")
+    time.sleep(5)
+    print("procs started")
+
+    cal = [msg for msg in lr if msg.which() == "liveCalibration"]
+    for msg in cal[:5]:
+      pm.send(msg.which(), msg.as_builder())
+
+    log_msgs = []
+    frame_idx = 0
+    for msg in tqdm(lr):
+      if msg.which() == "liveCalibrationd":
+        pm.send(msg.which(), msg.as_builder())
+      elif msg.which() == "frame":
+        f = msg.as_builder()
+        img = fr.get(frame_idx, pix_fmt="rgb24")[0][:, ::, -1]
+        f.frame.image = img.flatten().tobytes()
+        frame_idx += 1
+
+        pm.send(msg.which(), f)
+        log_msgs.append(messaging.recv_one(sm.sock['model']))
+
+        spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
+
+        if frame_idx >= fr.frame_count:
+          break
+  except KeyboardInterrupt:
+    pass
+
+  print("replay done")
+  spinner.close()
+  manager.kill_managed_process('modeld')
+  time.sleep(2)
+  manager.kill_managed_process('camerad')
+  return log_msgs
+
+
+if __name__ == "__main__":
+
+  update = "--update" in sys.argv
+
+  lr = LogReader(get_url(TEST_ROUTE, 0))
+  fr = FrameReader(get_url(TEST_ROUTE, 0, log_type="fcamera"))
+
+  log_msgs = camera_replay(list(lr), fr)
+
+  if update:
+    ref_commit = get_git_commit()
+    log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
+    save_log(log_fn, log_msgs)
+    with open("model_replay_ref_commit", "w") as f:
+      f.write(ref_commit)
+  else:
+    ref_commit = open("model_replay_ref_commit").read().strip()
+    log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
+    cmp_log = LogReader(BASE_URL + log_fn)
+    results: Any = {TEST_ROUTE: {}}
+    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid'])
+    diff1, diff2, failed = format_diff(results, ref_commit)
+
+    print(diff1)
+    with open("model_diff.txt", "w") as f:
+      f.write(diff2)
+
+    sys.exit(int(failed))
+

selfdrive/test/process_replay/model_replay_ref_commit

@@ -0,0 +1 @@
+18e43945403c4022b1de72237d89736e1a8ab4c7

selfdrive/test/process_replay/update_model.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from selfdrive.test.openpilotci_upload import upload_file
+from selfdrive.test.openpilotci import upload_file
 from selfdrive.test.process_replay.compare_logs import save_log
 from selfdrive.test.process_replay.test_processes import segments, get_segment
 from selfdrive.version import get_git_commit

selfdrive/test/process_replay/update_refs.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-from selfdrive.test.openpilotci_upload import upload_file
+from selfdrive.test.openpilotci import upload_file
 from selfdrive.test.process_replay.compare_logs import save_log
 from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS
 from selfdrive.test.process_replay.test_processes import segments, get_segment

tools/lib/auth_config.py

@@ -1,11 +1,15 @@
 import json
 import os
+from common.android import ANDROID
 from common.file_helpers import mkdirs_exists_ok
 
 class MissingAuthConfigError(Exception):
   pass
 
-CONFIG_DIR = os.path.expanduser('~/.comma')
+if ANDROID:
+  CONFIG_DIR = "/tmp/.comma"
+else:
+  CONFIG_DIR = os.path.expanduser('~/.comma')
 mkdirs_exists_ok(CONFIG_DIR)
 
 def get_token():