add tolerance to process replay compare (#1904)
This commit is contained in:
parent
95b0c69c12
commit
5139b827af
|
@ -5,6 +5,7 @@ import sys
|
|||
import numbers
|
||||
|
||||
import dictdiffer
|
||||
|
||||
if "CI" in os.environ:
|
||||
def tqdm(x):
|
||||
return x
|
||||
|
@ -13,6 +14,8 @@ else:
|
|||
|
||||
from tools.lib.logreader import LogReader
|
||||
|
||||
EPSILON = sys.float_info.epsilon
|
||||
|
||||
|
||||
def save_log(dest, log_msgs):
|
||||
dat = b""
|
||||
|
@ -49,7 +52,7 @@ def remove_ignored_fields(msg, ignore):
|
|||
return msg.as_reader()
|
||||
|
||||
|
||||
def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None):
|
||||
def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None, tolerance=None):
|
||||
if ignore_fields is None:
|
||||
ignore_fields = []
|
||||
|
||||
|
@ -70,7 +73,19 @@ def compare_logs(log1, log2, ignore_fields=None, ignore_msgs=None):
|
|||
if msg1_bytes != msg2_bytes:
|
||||
msg1_dict = msg1.to_dict(verbose=True)
|
||||
msg2_dict = msg2.to_dict(verbose=True)
|
||||
|
||||
tolerance = EPSILON if tolerance is None else tolerance
|
||||
dd = dictdiffer.diff(msg1_dict, msg2_dict, ignore=ignore_fields)
|
||||
|
||||
# Dictiffer only supports relative tolerance, we also want to check for absolute
|
||||
def outside_tolerance(diff):
|
||||
a, b = diff[2]
|
||||
if isinstance(a, numbers.Number) and isinstance(b, numbers.Number):
|
||||
return abs(a - b) > max(tolerance, tolerance * max(abs(a), abs(b)))
|
||||
return True
|
||||
|
||||
dd = list(filter(outside_tolerance, dd))
|
||||
|
||||
diff.extend(dd)
|
||||
return diff
|
||||
|
||||
|
|
|
@ -19,7 +19,7 @@ from common.params import Params
|
|||
from cereal.services import service_list
|
||||
from collections import namedtuple
|
||||
|
||||
ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback'])
|
||||
ProcessConfig = namedtuple('ProcessConfig', ['proc_name', 'pub_sub', 'ignore', 'init_callback', 'should_recv_callback', 'tolerance'])
|
||||
|
||||
|
||||
def wait_for_event(evt):
|
||||
|
@ -218,6 +218,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"],
|
||||
init_callback=fingerprint,
|
||||
should_recv_callback=None,
|
||||
tolerance=None,
|
||||
),
|
||||
ProcessConfig(
|
||||
proc_name="radard",
|
||||
|
@ -228,6 +229,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid", "radarState.cumLagMs"],
|
||||
init_callback=get_car_params,
|
||||
should_recv_callback=radar_rcv_callback,
|
||||
tolerance=None,
|
||||
),
|
||||
ProcessConfig(
|
||||
proc_name="plannerd",
|
||||
|
@ -238,6 +240,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid", "plan.processingDelay"],
|
||||
init_callback=get_car_params,
|
||||
should_recv_callback=None,
|
||||
tolerance=None,
|
||||
),
|
||||
ProcessConfig(
|
||||
proc_name="calibrationd",
|
||||
|
@ -248,6 +251,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid"],
|
||||
init_callback=get_car_params,
|
||||
should_recv_callback=calibration_rcv_callback,
|
||||
tolerance=None,
|
||||
),
|
||||
ProcessConfig(
|
||||
proc_name="dmonitoringd",
|
||||
|
@ -258,6 +262,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid"],
|
||||
init_callback=get_car_params,
|
||||
should_recv_callback=None,
|
||||
tolerance=None,
|
||||
),
|
||||
ProcessConfig(
|
||||
proc_name="locationd",
|
||||
|
@ -268,6 +273,7 @@ CONFIGS = [
|
|||
ignore=["logMonoTime", "valid"],
|
||||
init_callback=get_car_params,
|
||||
should_recv_callback=None,
|
||||
tolerance=1e-7, # Numpy gives different results based on CPU features after version 19
|
||||
),
|
||||
|
||||
]
|
||||
|
|
|
@ -71,7 +71,7 @@ def test_process(cfg, lr, cmp_log_fn, ignore_fields=None, ignore_msgs=None):
|
|||
segment = cmp_log_fn.split("/")[-1].split("_")[0]
|
||||
raise Exception("Route never enabled: %s" % segment)
|
||||
|
||||
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
|
||||
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs, cfg.tolerance)
|
||||
|
||||
|
||||
def format_diff(results, ref_commit):
|
||||
|
|
Loading…
Reference in a new issue