replay test (#205)

replay drives to test for safety violations. fails if any messages are blocked while controls are allowed

.circleci/config.yml

@@ -80,6 +80,19 @@ jobs:
           command: |
             docker run panda_build /bin/bash -c "cd /panda/boardesp; make user1.bin"
 
+  safety_replay:
+    machine:
+      docker_layer_caching: true
+    steps:
+      - checkout
+      - run:
+          name: Build image
+          command: "docker build -t panda_safety_replay -f tests/safety_replay/Dockerfile ."
+      - run:
+          name: Replay drives
+          command: |
+            docker run panda_safety_replay /bin/bash -c "cd /openpilot/panda/tests/safety_replay; PYTHONPATH=/openpilot ./test_safety_replay.py"
+
 workflows:
   version: 2
   main:
@@ -88,3 +101,4 @@ workflows:
       - misra-c2012
       - strict-compiler
       - build
+      - safety_replay

tests/safety/libpandasafety_py.py

@@ -40,6 +40,10 @@ int get_gas_interceptor_prev(void);
 void set_timer(int t);
 void reset_angle_control(void);
 
+void safety_rx_hook(CAN_FIFOMailBox_TypeDef *to_send);
+int safety_tx_hook(CAN_FIFOMailBox_TypeDef *to_push);
+int safety_set_mode(uint16_t  mode, int16_t param);
+
 void init_tests_toyota(void);
 void toyota_rx_hook(CAN_FIFOMailBox_TypeDef *to_push);
 int toyota_tx_hook(CAN_FIFOMailBox_TypeDef *to_send);

tests/safety_replay/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y make clang python python-pip git libarchive-dev libusb-1.0-0
+
+COPY tests/safety_replay/requirements.txt requirements.txt
+RUN pip install -r requirements.txt
+COPY tests/safety_replay/install_capnp.sh install_capnp.sh
+RUN ./install_capnp.sh
+
+RUN git clone https://github.com/commaai/openpilot.git || true
+WORKDIR /openpilot
+RUN git checkout 1e8098c140d924adb70167a216837fbf61d85f29
+RUN rm -rf /openpilot/panda
+COPY . /openpilot/panda
+
+WORKDIR /openpilot/panda/tests/safety_replay
+RUN git clone https://github.com/commaai/openpilot-tools.git openpilot_tools || true
+WORKDIR openpilot_tools
+RUN git checkout 3df301e5783d7dc37cf3b079e96ad07cd4f2d0c3

tests/safety_replay/install_capnp.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+apt-get install -y autoconf curl libtool
+curl -O https://capnproto.org/capnproto-c++-0.6.1.tar.gz
+tar xvf capnproto-c++-0.6.1.tar.gz
+cd capnproto-c++-0.6.1
+./configure --prefix=/usr/local CPPFLAGS=-DPIC CFLAGS=-fPIC CXXFLAGS=-fPIC LDFLAGS=-fPIC --disable-shared --enable-static
+make -j4
+make install
+
+cd ..
+git clone https://github.com/commaai/c-capnproto.git
+cd c-capnproto
+git checkout 2e625acacf58a5f5c8828d8453d1f8dacc700a96
+git submodule update --init --recursive
+autoreconf -f -i -s
+CFLAGS="-fPIC" ./configure --prefix=/usr/local
+make -j4
+make install
+

tests/safety_replay/replay_drive.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python2
+
+import sys
+import struct
+import panda.tests.safety.libpandasafety_py as libpandasafety_py
+from openpilot_tools.lib.logreader import LogReader
+
+safety_modes = {
+  "NOOUTPUT": 0,
+  "HONDA": 1,
+  "TOYOTA": 2,
+  "GM": 3,
+  "HONDA_BOSCH": 4,
+  "FORD": 5,
+  "CADILLAC": 6,
+  "HYUNDAI": 7,
+  "TESLA": 8,
+  "CHRYSLER": 9,
+  "SUBARU": 10,
+  "GM_ASCM": 0x1334,
+  "TOYOTA_IPAS": 0x1335,
+  "ALLOUTPUT": 0x1337,
+  "ELM327": 0xE327
+}
+
+# replay a drive to check for safety violations
+def replay_drive(lr, safety_mode, param):
+  safety = libpandasafety_py.libpandasafety
+
+  err = safety.safety_set_mode(safety_mode, param)
+  assert err == 0, "invalid safety mode: %d" % safety_mode
+
+  tx_tot, tx_blocked, tx_controls, tx_controls_blocked = 0, 0, 0, 0
+  blocked_addrs = set()
+
+  for msg in lr:
+    safety.set_timer((msg.logMonoTime & 0xFFFFFFFF)/1000)
+
+    if msg.which() == 'sendcan':
+      for canmsg in msg.sendcan:
+        # handle extended addresses
+        addr_shift = 3 if canmsg.address << 21 > 0xFFFFFFFF else 21
+
+        to_send = libpandasafety_py.ffi.new('CAN_FIFOMailBox_TypeDef *')
+        to_send[0].RIR = canmsg.address << addr_shift
+        to_send[0].RDTR = (canmsg.src & 0xF) << 4
+        to_send[0].RDHR = struct.unpack('<I', canmsg.dat.ljust(8, '\x00')[4:])[0]
+        to_send[0].RDLR = struct.unpack('<I', canmsg.dat.ljust(8, '\x00')[:4])[0]
+
+        sent = safety.safety_tx_hook(to_send)
+        if not sent:
+          tx_blocked += 1
+          tx_controls_blocked += safety.get_controls_allowed()
+          blocked_addrs.add(canmsg.address)
+        tx_controls += safety.get_controls_allowed()
+        tx_tot += 1
+    elif msg.which() == 'can':
+      for canmsg in msg.can:
+        # ignore msgs we sent
+        if canmsg.src >= 128:
+          continue
+
+        # handle extended addresses
+        addr_shift = 3 if canmsg.address << 21 > 0xFFFFFFFF else 21
+
+        to_push = libpandasafety_py.ffi.new('CAN_FIFOMailBox_TypeDef *')
+        to_push[0].RIR = canmsg.address << addr_shift
+        to_push[0].RDTR = (canmsg.src & 0xF) << 4
+        to_push[0].RDHR = struct.unpack('<I', canmsg.dat.ljust(8, '\x00')[4:])[0]
+        to_push[0].RDLR = struct.unpack('<I', canmsg.dat.ljust(8, '\x00')[:4])[0]
+        safety.safety_rx_hook(to_push)
+
+  print "total openpilot msgs:", tx_tot
+  print "total msgs with controls allowed:", tx_controls
+  print "blocked msgs:", tx_blocked
+  print "blocked with controls allowed:", tx_controls_blocked
+  print "blocked addrs:", blocked_addrs
+
+  return tx_controls_blocked == 0
+
+if __name__ == "__main__":
+  if sys.argv[2] in safety_modes:
+    mode = safety_modes[sys.argv[2]]
+  else:
+    mode = int(sys.argv[2])
+  param = 0 if len(sys.argv) < 4 else int(sys.argv[3])
+  lr = LogReader(sys.argv[1])
+
+  print "replaying drive %s with safety mode %d and param %d" % (sys.argv[1], mode, param)
+
+  replay_drive(lr, mode, param)

tests/safety_replay/requirements.txt

@@ -0,0 +1,8 @@
+aenum
+cffi==1.11.4
+libusb1==1.6.6
+numpy==1.14.5
+requests
+subprocess32
+libarchive
+pycapnp

tests/safety_replay/test_safety_replay.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python2
+
+import os
+import requests
+
+from replay_drive import replay_drive, safety_modes
+from openpilot_tools.lib.logreader import LogReader
+
+BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
+
+# (route, safety mode, param)
+logs = [
+  ("b0c9d2329ad1606b|2019-05-30--20-23-57.bz2", "HONDA", 0), # HONDA.CIVIC
+  ("38bfd238edecbcd7|2019-06-07--10-15-25.bz2", "TOYOTA", 66), # TOYOTA.PRIUS
+  ("f89c604cf653e2bf|2018-09-29--13-46-50.bz2", "GM", 0), # GM.VOLT
+  ("0375fdf7b1ce594d|2019-05-21--20-10-33.bz2", "HONDA_BOSCH", 1), # HONDA.ACCORD
+  ("02ec6bea180a4d36|2019-04-17--11-21-35.bz2", "HYUNDAI", 0), # HYUNDAI.SANTA_FE
+  ("03efb1fda29e30fe|2019-02-21--18-03-45.bz2", "CHRYSLER", 0), # CHRYSLER.PACIFICA_2018_HYBRID
+  ("791340bc01ed993d|2019-04-08--10-26-00.bz2", "SUBARU", 0), # SUBARU.IMPREZA
+]
+
+if __name__ == "__main__":
+  for route, _, _ in logs:
+    if not os.path.isfile(route):
+      with open(route, "w") as f:
+        f.write(requests.get(BASE_URL + route).content)
+
+  failed = []
+  for route, mode, param in logs:
+    lr = LogReader(route)
+    m = safety_modes.get(mode, mode)
+
+    print "\nreplaying %s with safety mode %d and param %s" % (route, m, param)
+    if not replay_drive(lr, m, int(param)):
+      failed.append(route)
+
+    for f in failed:
+      print "\n**** failed on %s ****" % f
+    assert len(failed) == 0, "\nfailed on %d logs" % len(failed)
+