Uploader speedup (#2214)

* use caching for getxattr

* fix some git issues

* Scheduled network checks

* attempt optimization

* Delete speed_test.py

* Style fixes

* Fix styling

* fix spaces

* fix spaces

* fix naming

* Update uploader.py

* Update mark_all_uploaded.py

* Add file to release

* Update selfdrive/loggerd/tools/mark_all_uploaded.py

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

release/files_common

@@ -315,6 +315,7 @@ selfdrive/loggerd/__init__.py
 selfdrive/loggerd/config.py
 selfdrive/loggerd/uploader.py
 selfdrive/loggerd/deleter.py
+selfdrive/loggerd/xattr_cache.py
 
 selfdrive/sensord/SConscript
 selfdrive/sensord/gpsd.cc

selfdrive/loggerd/tools/mark_all_uploaded.py

@@ -0,0 +1,9 @@
+import os
+from common.xattr import setxattr
+from selfdrive.loggerd.uploader import UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
+
+from selfdrive.loggerd.config import ROOT
+for folder in os.walk(ROOT):
+  for file1 in folder[2]:
+    full_path = os.path.join(folder[0], file1)
+    setxattr(full_path, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE)

selfdrive/loggerd/uploader.py

@@ -16,7 +16,7 @@ from cereal import log
 from common.hardware import HARDWARE
 from common.api import Api
 from common.params import Params
-from common.xattr import getxattr, setxattr
+from selfdrive.loggerd.xattr_cache import getxattr, setxattr
 from selfdrive.loggerd.config import ROOT
 from selfdrive.swaglog import cloudlog
 
@@ -26,6 +26,7 @@ UPLOAD_ATTR_VALUE = b'1'
 
 fake_upload = os.getenv("FAKEUPLOAD") is not None
 
+
 def raise_on_thread(t, exctype):
   '''Raises an exception in the threads with id tid'''
   for ctid, tobj in threading._active.items():
@@ -77,9 +78,9 @@ def is_on_hotspot():
   try:
     result = subprocess.check_output(["ifconfig", "wlan0"], stderr=subprocess.STDOUT, encoding='utf8')
     result = re.findall(r"inet addr:((\d+\.){3}\d+)", result)[0][0]
-    return (result.startswith('192.168.43.') or # android
-            result.startswith('172.20.10.') or # ios
-            result.startswith('10.0.2.')) # toyota entune
+    return (result.startswith('192.168.43.') or  # android
+            result.startswith('172.20.10.') or  # ios
+            result.startswith('10.0.2.'))  # toyota entune
   except Exception:
     return False
 
@@ -127,11 +128,11 @@ class Uploader():
           is_uploaded = True  # deleter could have deleted
         if is_uploaded:
           continue
-
         yield (name, key, fn)
 
   def next_file_to_upload(self, with_raw):
     upload_files = list(self.gen_upload_files())
+
     # try to upload qlog files first
     for name, key, fn in upload_files:
       if name in self.immediate_priority:
@@ -236,14 +237,19 @@ def uploader_fn(exit_event):
   uploader = Uploader(dongle_id, ROOT)
 
   backoff = 0.1
+  counter = 0
+  should_upload = False
   while not exit_event.is_set():
     offroad = params.get("IsOffroad") == b'1'
     allow_raw_upload = (params.get("IsUploadRawEnabled") != b"0") and offroad
-    on_hotspot = is_on_hotspot()
-    on_wifi = is_on_wifi()
-    should_upload = on_wifi and not on_hotspot
+    check_network = (counter % 12 == 0 if offroad else True)
+    if check_network:
+      on_hotspot = is_on_hotspot()
+      on_wifi = is_on_wifi()
+      should_upload = on_wifi and not on_hotspot
 
     d = uploader.next_file_to_upload(with_raw=allow_raw_upload and should_upload)
+    counter += 1
     if d is None:  # Nothing to upload
       time.sleep(60 if offroad else 5)
       continue
@@ -264,5 +270,6 @@ def uploader_fn(exit_event):
 def main():
   uploader_fn(threading.Event())
 
+
 if __name__ == "__main__":
   main()

selfdrive/loggerd/xattr_cache.py

@@ -0,0 +1,13 @@
+from common.xattr import getxattr as getattr1
+from common.xattr import setxattr as setattr1
+
+cached_attributes = {}
+def getxattr(path, attr_name):
+  if (path, attr_name) not in cached_attributes:
+    response = getattr1(path, attr_name)
+    cached_attributes[(path, attr_name)] = response
+  return cached_attributes[(path, attr_name)]
+
+def setxattr(path, attr_name, attr_value):
+  cached_attributes.pop((path, attr_name), None)
+  return setattr1(path, attr_name, attr_value)