spacecruft
/
nopenpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Process replay enhancements (#1153) 

* process replay: exclude processes and cars
pull/1163/head
Adeeb 2020-02-26 11:31:27 -08:00 committed by GitHub
parent d7a4bb3853
commit 796d5c8cca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 145 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
selfdrive/car/car_helpers.py
View File

@ -56,7 +56,8 @@ def _get_interface_names():
# imports from directory selfdrive/car/<name>/
interfaces = load_interfaces(_get_interface_names())
interface_names = _get_interface_names()
interfaces = load_interfaces(interface_names)
def only_toyota_left(candidate_cars):

21
selfdrive/test/process_replay/compare_logs.py
View File

@ -2,6 +2,7 @@
import bz2
import os
import sys
import numbers
import dictdiffer
if "CI" in os.environ:
@ -22,7 +23,7 @@ def save_log(dest, log_msgs):
def remove_ignored_fields(msg, ignore):
msg = msg.as_builder()
for key, val in ignore:
for key in ignore:
attr = msg
keys = key.split(".")
if msg.which() not in key and len(keys) > 1:
@ -34,21 +35,29 @@ def remove_ignored_fields(msg, ignore):
except:
break
else:
v = getattr(attr, keys[-1])
if isinstance(v, bool):
val = False
elif isinstance(v, numbers.Number):
val = 0
else:
raise NotImplementedError
setattr(attr, keys[-1], val)
return msg.as_reader()
def compare_logs(log1, log2, ignore=[]):
def compare_logs(log1, log2, ignore_fields=[], ignore_msgs=[]):
filter_msgs = lambda m: m.which() not in ignore_msgs
log1, log2 = [list(filter(filter_msgs, log)) for log in (log1, log2)]
assert len(log1) == len(log2), "logs are not same length: " + str(len(log1)) + " VS " + str(len(log2))
ignore_fields = [k for k, v in ignore]
diff = []
for msg1, msg2 in tqdm(zip(log1, log2)):
if msg1.which() != msg2.which():
print(msg1, msg2)
assert False, "msgs not aligned between logs"
raise Exception("msgs not aligned between logs")
msg1_bytes = remove_ignored_fields(msg1, ignore).as_builder().to_bytes()
msg2_bytes = remove_ignored_fields(msg2, ignore).as_builder().to_bytes()
msg1_bytes = remove_ignored_fields(msg1, ignore_fields).as_builder().to_bytes()
msg2_bytes = remove_ignored_fields(msg2, ignore_fields).as_builder().to_bytes()
if msg1_bytes != msg2_bytes:
msg1_dict = msg1.to_dict(verbose=True)

10
selfdrive/test/process_replay/process_replay.py
View File

@ -200,7 +200,7 @@ CONFIGS = [
"thermal": [], "health": [], "liveCalibration": [], "dMonitoringState": [], "plan": [], "pathPlan": [], "gpsLocation": [],
"model": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("controlsState.startMonoTime", 0), ("controlsState.cumLagMs", 0)],
ignore=["logMonoTime", "valid", "controlsState.startMonoTime", "controlsState.cumLagMs"],
init_callback=fingerprint,
should_recv_callback=None,
),
@ -210,7 +210,7 @@ CONFIGS = [
"can": ["radarState", "liveTracks"],
"liveParameters": [], "controlsState": [], "model": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("radarState.cumLagMs", 0)],
ignore=["logMonoTime", "valid", "radarState.cumLagMs"],
init_callback=get_car_params,
should_recv_callback=radar_rcv_callback,
),
@ -220,7 +220,7 @@ CONFIGS = [
"model": ["pathPlan"], "radarState": ["plan"],
"carState": [], "controlsState": [], "liveParameters": [],
},
ignore=[("logMonoTime", 0), ("valid", True), ("plan.processingDelay", 0)],
ignore=["logMonoTime", "valid", "plan.processingDelay"],
init_callback=get_car_params,
should_recv_callback=None,
),
@ -229,7 +229,7 @@ CONFIGS = [
pub_sub={
"cameraOdometry": ["liveCalibration"]
},
ignore=[("logMonoTime", 0), ("valid", True)],
ignore=["logMonoTime", "valid"],
init_callback=get_car_params,
should_recv_callback=calibration_rcv_callback,
),
@ -239,7 +239,7 @@ CONFIGS = [
"driverState": ["dMonitoringState"],
"liveCalibration": [], "carState": [], "model": [], "gpsLocation": [],
},
ignore=[("logMonoTime", 0), ("valid", True)],
ignore=["logMonoTime", "valid"],
init_callback=get_car_params,
should_recv_callback=None,
),

196
selfdrive/test/process_replay/test_processes.py
View File

@ -1,117 +1,167 @@
#!/usr/bin/env python3
import argparse
import os
import requests
import sys
import tempfile
from selfdrive.car.car_helpers import interface_names
from selfdrive.test.process_replay.compare_logs import compare_logs
from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS
from tools.lib.logreader import LogReader
segments = [
"0375fdf7b1ce594d|2019-06-13--08-32-25--3", # HONDA.ACCORD
"99c94dc769b5d96e|2019-08-03--14-19-59--2", # HONDA.CIVIC
"cce908f7eb8db67d|2019-08-02--15-09-51--3", # TOYOTA.COROLLA_TSS2
"7ad88f53d406b787|2019-07-09--10-18-56--8", # GM.VOLT
"704b2230eb5190d6|2019-07-06--19-29-10--0", # HYUNDAI.KIA_SORENTO
"b6e1317e1bfbefa6|2019-07-06--04-05-26--5", # CHRYSLER.JEEP_CHEROKEE
"7873afaf022d36e2|2019-07-03--18-46-44--0", # SUBARU.IMPREZA
"b0c9d2329ad1606b|2020-02-19--16-29-36--7", # VW.GOLF
("HONDA", "0375fdf7b1ce594d|2019-06-13--08-32-25--3"), # HONDA.ACCORD
("HONDA", "99c94dc769b5d96e|2019-08-03--14-19-59--2"), # HONDA.CIVIC
("TOYOTA", "cce908f7eb8db67d|2019-08-02--15-09-51--3"), # TOYOTA.COROLLA_TSS2
("GM", "7ad88f53d406b787|2019-07-09--10-18-56--8"), # GM.VOLT
("HYUNDAI", "704b2230eb5190d6|2019-07-06--19-29-10--0"), # HYUNDAI.KIA_SORENTO
("CHRYSLER", "b6e1317e1bfbefa6|2019-07-06--04-05-26--5"), # CHRYSLER.JEEP_CHEROKEE
("SUBARU", "7873afaf022d36e2|2019-07-03--18-46-44--0"), # SUBARU.IMPREZA
("VOLKSWAGEN", "b0c9d2329ad1606b|2020-02-19--16-29-36--7"), # VW.GOLF
]
# ford doesn't need to be tested until a full port is done
excluded_interfaces = ["mock", "ford"]
BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/"
# run the full test (including checks) when no args given
FULL_TEST = len(sys.argv) <= 1
def get_segment(segment_name):
route_name, segment_num = segment_name.rsplit("--", 1)
rlog_url = "https://commadataci.blob.core.windows.net/openpilotci/%s/%s/rlog.bz2" \
% (route_name.replace("|", "/"), segment_num)
r = requests.get(rlog_url)
if r.status_code != 200:
return None
rlog_url = BASE_URL + "%s/%s/rlog.bz2" % (route_name.replace("|", "/"), segment_num)
req = requests.get(rlog_url)
assert req.status_code == 200, ("Failed to download log for %s" % segment_name)
with tempfile.NamedTemporaryFile(delete=False, suffix=".bz2") as f:
f.write(r.content)
f.write(req.content)
return f.name
def test_process(cfg, lr, cmp_log_fn, ignore_fields=[], ignore_msgs=[]):
if not os.path.isfile(cmp_log_fn):
req = requests.get(BASE_URL + os.path.basename(cmp_log_fn))
assert req.status_code == 200, ("Failed to download %s" % cmp_log_fn)
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(cmp_log_fn))
log_msgs = replay_process(cfg, lr)
# check to make sure openpilot is engaged in the route
# TODO: update routes so enable check can run
# failed enable check: honda bosch, hyundai, chrysler, and subaru
if cfg.proc_name == "controlsd" and FULL_TEST and False:
for msg in log_msgs:
if msg.which() == "controlsState":
if msg.controlsState.active:
break
else:
segment = cmp_log_fn.split("/")[-1].split("_")[0]
raise Exception("Route never enabled: %s" % segment)
return compare_logs(cmp_log_msgs, log_msgs, ignore_fields+cfg.ignore, ignore_msgs)
def format_diff(results, ref_commit):
diff1, diff2 = "", ""
diff2 += "***** tested against commit %s *****\n" % ref_commit
failed = False
for segment, result in list(results.items()):
diff1 += "***** results for segment %s *****\n" % segment
diff2 += "***** differences for segment %s *****\n" % segment
for proc, diff in list(result.items()):
diff1 += "\t%s\n" % proc
diff2 += "*** process: %s ***\n" % proc
if isinstance(diff, str):
diff1 += "\t\t%s\n" % diff
failed = True
elif len(diff):
cnt = {}
for d in diff:
diff2 += "\t%s\n" % str(d)
k = str(d[1])
cnt[k] = 1 if k not in cnt else cnt[k] + 1
for k, v in sorted(cnt.items()):
diff1 += "\t\t%s: %s\n" % (k, v)
failed = True
return diff1, diff2, failed
if __name__ == "__main__":
process_replay_dir = os.path.dirname(os.path.abspath(__file__))
ref_commit_fn = os.path.join(process_replay_dir, "ref_commit")
parser = argparse.ArgumentParser(description="Regression test to identify changes in a process's output")
if not os.path.isfile(ref_commit_fn):
# whitelist has precedence over blacklist in case both are defined
parser.add_argument("--whitelist-procs", type=str, nargs="*", default=[],
help="Whitelist given processes from the test (e.g. controlsd)")
parser.add_argument("--whitelist-cars", type=str, nargs="*", default=[],
help="Whitelist given cars from the test (e.g. HONDA)")
parser.add_argument("--blacklist-procs", type=str, nargs="*", default=[],
help="Blacklist given processes from the test (e.g. controlsd)")
parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
help="Blacklist given cars from the test (e.g. HONDA)")
parser.add_argument("--ignore-fields", type=str, nargs="*", default=[],
help="Extra fields or msgs to ignore (e.g. carState.events)")
parser.add_argument("--ignore-msgs", type=str, nargs="*", default=[],
help="Msgs to ignore (e.g. carEvents)")
args = parser.parse_args()
cars_whitelisted = len(args.whitelist_cars) > 0
procs_whitelisted = len(args.whitelist_procs) > 0
process_replay_dir = os.path.dirname(os.path.abspath(__file__))
try:
ref_commit = open(os.path.join(process_replay_dir, "ref_commit")).read().strip()
except:
print("couldn't find reference commit")
sys.exit(1)
ref_commit = open(ref_commit_fn).read().strip()
print("***** testing against commit %s *****" % ref_commit)
# check to make sure all car brands are tested
if FULL_TEST:
tested_cars = set(c.lower() for c, _ in segments)
untested = (set(interface_names) - set(excluded_interfaces)) - tested_cars
assert len(untested) == 0, "Cars missing routes: %s" % (str(untested))
results = {}
for segment in segments:
for car_brand, segment in segments:
if (cars_whitelisted and car_brand.upper() not in args.whitelist_cars) or \
(not cars_whitelisted and car_brand.upper() in args.blacklist_cars):
continue
print("***** testing route segment %s *****\n" % segment)
results[segment] = {}
rlog_fn = get_segment(segment)
if rlog_fn is None:
print("failed to get segment %s" % segment)
sys.exit(1)
lr = LogReader(rlog_fn)
for cfg in CONFIGS:
log_msgs = replay_process(cfg, lr)
if (procs_whitelisted and cfg.proc_name not in args.whitelist_procs) or \
(not procs_whitelisted and cfg.proc_name in args.blacklist_procs):
continue
log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
if not os.path.isfile(log_fn):
url = "https://commadataci.blob.core.windows.net/openpilotci/"
req = requests.get(url + os.path.basename(log_fn))
if req.status_code != 200:
results[segment][cfg.proc_name] = "failed to download comparison log"
continue
with tempfile.NamedTemporaryFile(suffix=".bz2") as f:
f.write(req.content)
f.flush()
f.seek(0)
cmp_log_msgs = list(LogReader(f.name))
else:
cmp_log_msgs = list(LogReader(log_fn))
diff = compare_logs(cmp_log_msgs, log_msgs, cfg.ignore)
results[segment][cfg.proc_name] = diff
cmp_log_fn = os.path.join(process_replay_dir, "%s_%s_%s.bz2" % (segment, cfg.proc_name, ref_commit))
results[segment][cfg.proc_name] = test_process(cfg, lr, cmp_log_fn, args.ignore_fields, args.ignore_msgs)
os.remove(rlog_fn)
failed = False
diff1, diff2, failed = format_diff(results, ref_commit)
with open(os.path.join(process_replay_dir, "diff.txt"), "w") as f:
f.write("***** tested against commit %s *****\n" % ref_commit)
f.write(diff2)
print(diff1)
for segment, result in list(results.items()):
f.write("***** differences for segment %s *****\n" % segment)
print("***** results for segment %s *****" % segment)
for proc, diff in list(result.items()):
f.write("*** process: %s ***\n" % proc)
print("\t%s" % proc)
if isinstance(diff, str):
print("\t\t%s" % diff)
failed = True
elif len(diff):
cnt = {}
for d in diff:
f.write("\t%s\n" % str(d))
k = str(d[1])
cnt[k] = 1 if k not in cnt else cnt[k] + 1
for k, v in sorted(cnt.items()):
print("\t\t%s: %s" % (k, v))
failed = True
if failed:
print("TEST FAILED")
else:
print("TEST SUCCEEDED")
print("TEST", "FAILED" if failed else "SUCCEEDED")
print("\n\nTo update the reference logs for this test run:")
print("./update_refs.py")