add tolerance to process replay compare (#1904)

selfdrive/test/process_replay/compare_logs.py

@@ -5,6 +5,7 @@ import sys
 import numbers
 
 import dictdiffer
+
 if "CI" in os.environ:
   def tqdm(x):
     return x
@@ -13,6 +14,8 @@ else:
 
 from tools.lib.logreader import LogReader
 
+EPSILON = sys.float_info.epsilon
+
 
 def save_log(dest, log_msgs):
   dat = b""
@@ -49,7 +52,7 @@ def remove_ignored_fields(msg, ignore):
   return msg.as_reader()
 
 
-def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None):
+def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None):
   if ignore_fields is None:
     ignore_fields = []
 
@@ -70,7 +73,19 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None):
     if msg1_bytes != msg2_bytes:
       msg1_dict = msg1.to_dict(verbose=True)
       msg2_dict = msg2.to_dict(verbose=True)
+
+      tolerance = EPSILON if tolerance is None else tolerance
       dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
+
+      # Dictiffer only supports relative tolerance, we also want to check for absolute
+      def outside_tolerance(diff):
+        a, b = diff[2]
+        if isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
+          return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
+        return True
+
+      dd = list(filter(outside_tolerance, dd))
+
       diff.extend(dd)
   return diff
 

selfdrive/test/process_replay/process_replay.py

@@ -19,7 +19,7 @@ from common.params import Params
 from cereal.services import service_list
 from collections import namedtuple
 
-ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback'])
+ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback', 'tolerance'])
 
 
 def wait_for_event(evt):
@@ -218,6 +218,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"],
     init_callback=fingerprint,
     should_recv_callback=None,
+    tolerance=None,
   ),
   ProcessConfig(
     proc_name="radard",
@@ -228,6 +229,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid", "radarState.cumLagMs"],
     init_callback=get_car_params,
     should_recv_callback=radar_rcv_callback,
+    tolerance=None,
   ),
   ProcessConfig(
     proc_name="plannerd",
@@ -238,6 +240,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid", "plan.processingDelay"],
     init_callback=get_car_params,
     should_recv_callback=None,
+    tolerance=None,
   ),
   ProcessConfig(
     proc_name="calibrationd",
@@ -248,6 +251,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=calibration_rcv_callback,
+    tolerance=None,
   ),
   ProcessConfig(
     proc_name="dmonitoringd",
@@ -258,6 +262,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=None,
+    tolerance=None,
   ),
   ProcessConfig(
     proc_name="locationd",
@@ -268,6 +273,7 @@ CONFIGS = [
     ignore=["logMonoTime", "valid"],
     init_callback=get_car_params,
     should_recv_callback=None,
+    tolerance=1e-7,  # Numpy gives different results based on CPU features after version 19
   ),
 
 ]

selfdrive/test/process_replay/test_processes.py

@@ -71,7 +71,7 @@ def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None):
       segment = cmp_log_fn.split("/")[-1].split("_")[0]
       raise Exception("Route never enabled: %s" % segment)
 
-  return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
+  return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs, cfg.tolerance)
 
 
 def format_diff(results, ref_commit):