add modelV2 to model replay (#2725)

* add modelV2 to model replay

* update refs

* ignore fields for modelv2

* execution time

* fix compare logs

Co-authored-by: Comma Device <device@comma.ai>

selfdrive/test/openpilotci.py

@@ -16,7 +16,7 @@ def upload_file(path, name):
 
   sas_token = None
   if os.path.isfile(TOKEN_PATH):
-    sas_token = open(TOKEN_PATH).read().strip
+    sas_token = open(TOKEN_PATH).read().strip()
 
   if sas_token is None:
     sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw \

selfdrive/test/process_replay/camera_replay.py

@@ -35,9 +35,10 @@ def replace_calib(msg, calib):
 def camera_replay(lr, fr, desire=None, calib=None):
 
   spinner = Spinner()
+  spinner.update("starting model replay")
 
   pm = messaging.PubMaster(['frame', 'liveCalibration', 'pathPlan'])
-  sm = messaging.SubMaster(['model'])
+  sm = messaging.SubMaster(['model', 'modelV2'])
 
   # TODO: add dmonitoringmodeld
   print("preparing procs")
@@ -48,6 +49,7 @@ def camera_replay(lr, fr, desire=None, calib=None):
     manager.start_managed_process("camerad")
     manager.start_managed_process("modeld")
     time.sleep(5)
+    sm.update(1000)
     print("procs started")
 
     desires_by_index = {v:k for k,v in log.PathPlan.Desire.schema.enumerants.items()}
@@ -76,6 +78,7 @@ def camera_replay(lr, fr, desire=None, calib=None):
         pm.send(msg.which(), f)
         with Timeout(seconds=15):
           log_msgs.append(messaging.recv_one(sm.sock['model']))
+          log_msgs.append(messaging.recv_one(sm.sock['modelV2']))
 
         spinner.update("modeld replay %d/%d" % (frame_idx, fr.frame_count))
 
@@ -108,8 +111,11 @@ if __name__ == "__main__":
     ref_commit = open(ref_commit_fn).read().strip()
     log_fn = "%s_%s_%s.bz2" % (TEST_ROUTE, "model", ref_commit)
     cmp_log = LogReader(BASE_URL + log_fn)
+
+    ignore = ['logMonoTime', 'valid', 'model.frameDropPerc', 'model.modelExecutionTime',
+              'modelV2.frameDropPerc', 'modelV2.modelExecutionTime']
     results: Any = {TEST_ROUTE: {}}
-    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=['logMonoTime', 'valid', 'model.frameDropPerc', 'model.modelExecutionTime'])
+    results[TEST_ROUTE]["modeld"] = compare_logs(cmp_log, log_msgs, ignore_fields=ignore)
     diff1, diff2, failed = format_diff(results, ref_commit)
 
     print(diff1)

selfdrive/test/process_replay/compare_logs.py

@@ -31,7 +31,7 @@ def remove_ignored_fields(msg, ignore):
   for key in ignore:
     attr = msg
     keys = key.split(".")
-    if msg.which() not in key and len(keys) > 1:
+    if msg.which() != keys[0] and len(keys) > 1:
       continue
 
     for k in keys[:-1]:

selfdrive/test/process_replay/model_replay_ref_commit

@@ -1 +1 @@
-f040d87da71d8d9b29000b0eeea43d4f932137dd
+857a00fc1093422907d49fc8647655de498f195b