diff --git a/common/api/__init__.py b/common/api/__init__.py index 5b1d66cf7..8b83dfc64 100644 --- a/common/api/__init__.py +++ b/common/api/__init__.py @@ -3,7 +3,7 @@ import os import requests from datetime import datetime, timedelta from common.basedir import PERSIST -from selfdrive.version import version +from selfdrive.version import get_version API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com') @@ -34,13 +34,13 @@ class Api(): if isinstance(token, bytes): token = token.decode('utf8') return token - + def api_get(endpoint, method='GET', timeout=None, access_token=None, **params): headers = {} if access_token is not None: - headers['Authorization'] = "JWT "+access_token + headers['Authorization'] = "JWT " + access_token - headers['User-Agent'] = "openpilot-" + version + headers['User-Agent'] = "openpilot-" + get_version() return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params) diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index 779045bf9..5e462a42f 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -30,7 +30,7 @@ from selfdrive.hardware import HARDWARE, PC from selfdrive.loggerd.config import ROOT from selfdrive.loggerd.xattr_cache import getxattr, setxattr from selfdrive.swaglog import cloudlog, SWAGLOG_DIR -from selfdrive.version import version, get_version, get_git_remote, get_git_branch, get_git_commit +from selfdrive.version import get_version, get_origin, get_short_branch, get_commit ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai') HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) @@ -176,9 +176,9 @@ def getMessage(service=None, timeout=1000): def getVersion(): return { "version": get_version(), - "remote": get_git_remote(), - "branch": get_git_branch(), - "commit": get_git_commit(), + "remote": get_origin(), + "branch": get_short_branch(), + "commit": get_commit(), } @@ -551,7 +551,7 @@ def main(): except socket.timeout: try: r = requests.get("http://api.commadotai.com/v1/me", allow_redirects=False, - headers={"User-Agent": f"openpilot-{version}"}, timeout=15.0) + headers={"User-Agent": f"openpilot-{get_version()}"}, timeout=15.0) if r.status_code == 302 and r.headers['Location'].startswith("http://u.web2go.com"): params.put_bool("PrimeRedirected", True) except Exception: diff --git a/selfdrive/athena/manage_athenad.py b/selfdrive/athena/manage_athenad.py index 7bb717e07..fa95eacd8 100755 --- a/selfdrive/athena/manage_athenad.py +++ b/selfdrive/athena/manage_athenad.py @@ -6,7 +6,7 @@ from multiprocessing import Process from common.params import Params from selfdrive.manager.process import launcher from selfdrive.swaglog import cloudlog -from selfdrive.version import version, dirty +from selfdrive.version import get_version, get_dirty ATHENA_MGR_PID_PARAM = "AthenadPid" @@ -14,7 +14,7 @@ ATHENA_MGR_PID_PARAM = "AthenadPid" def main(): params = Params() dongle_id = params.get("DongleId").decode('utf-8') - cloudlog.bind_global(dongle_id=dongle_id, version=version, dirty=dirty) + cloudlog.bind_global(dongle_id=dongle_id, version=get_version(), dirty=get_dirty()) try: while 1: diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 980b207fa..786b00fa2 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,7 +1,7 @@ import os from common.params import Params from common.basedir import BASEDIR -from selfdrive.version import comma_remote, tested_branch +from selfdrive.version import get_comma_remote, get_tested_branch from selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars from selfdrive.car.vin import get_vin, VIN_UNKNOWN from selfdrive.car.fw_versions import get_fw_versions, match_fw_to_car @@ -14,7 +14,7 @@ EventName = car.CarEvent.EventName def get_startup_event(car_recognized, controller_available, fw_seen): - if comma_remote and tested_branch: + if get_comma_remote() and get_tested_branch(): event = EventName.startup else: event = EventName.startupMaster diff --git a/selfdrive/crash.py b/selfdrive/crash.py index f85d7c0e6..e42b7532b 100644 --- a/selfdrive/crash.py +++ b/selfdrive/crash.py @@ -1,6 +1,6 @@ """Install exception handler for process crash.""" from selfdrive.swaglog import cloudlog -from selfdrive.version import version +from selfdrive.version import get_version import sentry_sdk from sentry_sdk.integrations.threading import ThreadingIntegration @@ -24,4 +24,4 @@ def bind_extra(**kwargs) -> None: def init() -> None: sentry_sdk.init("https://a8dc76b5bfb34908a601d67e2aa8bcf9@o33823.ingest.sentry.io/77924", default_integrations=False, integrations=[ThreadingIntegration(propagate_hub=True)], - release=version) + release=get_version()) diff --git a/selfdrive/loggerd/tests/test_loggerd.py b/selfdrive/loggerd/tests/test_loggerd.py index 7ee97f511..021eea5c8 100755 --- a/selfdrive/loggerd/tests/test_loggerd.py +++ b/selfdrive/loggerd/tests/test_loggerd.py @@ -18,7 +18,7 @@ from selfdrive.hardware import PC, TICI from selfdrive.loggerd.config import ROOT from selfdrive.manager.process_config import managed_processes from selfdrive.test.helpers import with_processes -from selfdrive.version import version as VERSION +from selfdrive.version import get_version from tools.lib.logreader import LogReader SentinelType = log.Sentinel.SentinelType @@ -95,7 +95,7 @@ class TestLoggerd(unittest.TestCase): initData = lr[0].initData self.assertTrue(initData.dirty != bool(os.environ["CLEAN"])) - self.assertEqual(initData.version, VERSION) + self.assertEqual(initData.version, get_version()) if os.path.isfile("/proc/cmdline"): with open("/proc/cmdline") as f: diff --git a/selfdrive/manager/build.py b/selfdrive/manager/build.py index c4fd4746e..04769526f 100755 --- a/selfdrive/manager/build.py +++ b/selfdrive/manager/build.py @@ -12,7 +12,7 @@ from common.spinner import Spinner from common.text_window import TextWindow from selfdrive.hardware import TICI from selfdrive.swaglog import cloudlog, add_file_handler -from selfdrive.version import dirty +from selfdrive.version import get_dirty MAX_CACHE_SIZE = 2e9 CACHE_DIR = Path("/data/scons_cache" if TICI else "/tmp/scons_cache") @@ -98,4 +98,4 @@ def build(spinner, dirty=False): if __name__ == "__main__" and not PREBUILT: spinner = Spinner() spinner.update_progress(0, 100) - build(spinner, dirty) + build(spinner, get_dirty()) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 4c70a3851..66de8eccc 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -18,12 +18,13 @@ from selfdrive.manager.process import ensure_running from selfdrive.manager.process_config import managed_processes from selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID from selfdrive.swaglog import cloudlog, add_file_handler -from selfdrive.version import dirty, get_git_commit, version, origin, branch, commit, \ - terms_version, training_version, comma_remote, \ - get_git_branch, get_git_remote +from selfdrive.version import get_dirty, get_commit, get_version, get_origin, get_branch, \ + terms_version, training_version, get_comma_remote + sys.path.append(os.path.join(BASEDIR, "pyextra")) + def manager_init(): # update system time from panda set_time(cloudlog) @@ -69,12 +70,12 @@ def manager_init(): print("WARNING: failed to make /dev/shm") # set version params - params.put("Version", version) + params.put("Version", get_version()) params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - params.put("GitCommit", get_git_commit(default="")) - params.put("GitBranch", get_git_branch(default="")) - params.put("GitRemote", get_git_remote(default="")) + params.put("GitCommit", get_commit(default="")) + params.put("GitBranch", get_branch(default="")) + params.put("GitRemote", get_origin(default="")) # set dongle id reg_res = register(show_spinner=True) @@ -85,16 +86,16 @@ def manager_init(): raise Exception(f"Registration failed for device {serial}") os.environ['DONGLE_ID'] = dongle_id # Needed for swaglog - if not dirty: + if not get_dirty(): os.environ['CLEAN'] = '1' - cloudlog.bind_global(dongle_id=dongle_id, version=version, dirty=dirty, + cloudlog.bind_global(dongle_id=dongle_id, version=get_version(), dirty=get_dirty(), device=HARDWARE.get_device_type()) - if comma_remote and not (os.getenv("NOLOG") or os.getenv("NOCRASH") or PC): + if get_comma_remote() and not (os.getenv("NOLOG") or os.getenv("NOCRASH") or PC): crash.init() crash.bind_user(id=dongle_id) - crash.bind_extra(dirty=dirty, origin=origin, branch=branch, commit=commit, + crash.bind_extra(dirty=get_dirty(), origin=get_origin(), branch=get_branch(), commit=get_commit(), device=HARDWARE.get_device_type()) diff --git a/selfdrive/test/process_replay/model_replay.py b/selfdrive/test/process_replay/model_replay.py index 3b03bc840..16ffea3c7 100755 --- a/selfdrive/test/process_replay/model_replay.py +++ b/selfdrive/test/process_replay/model_replay.py @@ -17,7 +17,7 @@ from selfdrive.manager.process_config import managed_processes from selfdrive.test.openpilotci import BASE_URL, get_url from selfdrive.test.process_replay.compare_logs import compare_logs, save_log from selfdrive.test.process_replay.test_processes import format_diff -from selfdrive.version import get_git_commit +from selfdrive.version import get_commit from tools.lib.framereader import FrameReader from tools.lib.logreader import LogReader @@ -169,7 +169,7 @@ if __name__ == "__main__": print("Uploading new refs") - new_commit = get_git_commit() + new_commit = get_commit() log_fn = get_log_fn(new_commit) save_log(log_fn, log_msgs) try: diff --git a/selfdrive/test/process_replay/update_refs.py b/selfdrive/test/process_replay/update_refs.py index fe5212d2c..96feb0d53 100755 --- a/selfdrive/test/process_replay/update_refs.py +++ b/selfdrive/test/process_replay/update_refs.py @@ -6,7 +6,7 @@ from selfdrive.test.openpilotci import upload_file, get_url from selfdrive.test.process_replay.compare_logs import save_log from selfdrive.test.process_replay.process_replay import replay_process, CONFIGS from selfdrive.test.process_replay.test_processes import segments -from selfdrive.version import get_git_commit +from selfdrive.version import get_commit from tools.lib.logreader import LogReader if __name__ == "__main__": @@ -16,7 +16,7 @@ if __name__ == "__main__": process_replay_dir = os.path.dirname(os.path.abspath(__file__)) ref_commit_fn = os.path.join(process_replay_dir, "ref_commit") - ref_commit = get_git_commit() + ref_commit = get_commit() if ref_commit is None: raise Exception("couldn't get ref commit") with open(ref_commit_fn, "w") as f: diff --git a/selfdrive/thermald/thermald.py b/selfdrive/thermald/thermald.py index 114110891..eb3e59426 100755 --- a/selfdrive/thermald/thermald.py +++ b/selfdrive/thermald/thermald.py @@ -22,7 +22,7 @@ from selfdrive.hardware import EON, TICI, PC, HARDWARE from selfdrive.loggerd.config import get_available_percent from selfdrive.swaglog import cloudlog from selfdrive.thermald.power_monitoring import PowerMonitoring -from selfdrive.version import tested_branch, terms_version, training_version +from selfdrive.version import get_tested_branch, terms_version, training_version ThermalStatus = log.DeviceState.ThermalStatus NetworkType = log.DeviceState.NetworkType @@ -343,7 +343,7 @@ def thermald_thread(): last_update_exception = params.get("LastUpdateException", encoding='utf8') if update_failed_count > 15 and last_update_exception is not None: - if tested_branch: + if get_tested_branch(): extra_text = "Ensure the software is correctly installed" else: extra_text = last_update_exception diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py index fb7a1a2c9..a301725ba 100755 --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -15,7 +15,7 @@ from common.file_helpers import mkdirs_exists_ok from selfdrive.hardware import TICI, HARDWARE from selfdrive.loggerd.config import ROOT from selfdrive.swaglog import cloudlog -from selfdrive.version import branch, commit, dirty, origin, version +from selfdrive.version import get_branch, get_commit, get_dirty, get_origin, get_version MAX_SIZE = 100000 * 10 # mal size is 40-100k, allow up to 1M if TICI: @@ -109,7 +109,7 @@ def report_tombstone_android(fn): clean_path = executable.replace('./', '').replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") - new_fn = f"{date}_{commit[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] + new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(ROOT, "crash") mkdirs_exists_ok(crashlog_dir) @@ -183,7 +183,7 @@ def report_tombstone_apport(fn): clean_path = path.replace('/', '_') date = datetime.datetime.now().strftime("%Y-%m-%d--%H-%M-%S") - new_fn = f"{date}_{commit[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] + new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] crashlog_dir = os.path.join(ROOT, "crash") mkdirs_exists_ok(crashlog_dir) @@ -203,14 +203,14 @@ def main(): sentry_sdk.utils.MAX_STRING_LENGTH = 8192 sentry_sdk.init("https://a40f22e13cbc4261873333c125fc9d38@o33823.ingest.sentry.io/157615", - default_integrations=False, release=version) + default_integrations=False, release=get_version()) dongle_id = Params().get("DongleId", encoding='utf-8') sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", dirty) - sentry_sdk.set_tag("origin", origin) - sentry_sdk.set_tag("branch", branch) - sentry_sdk.set_tag("commit", commit) + sentry_sdk.set_tag("dirty", get_dirty()) + sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_tag("branch", get_branch()) + sentry_sdk.set_tag("commit", get_commit()) sentry_sdk.set_tag("device", HARDWARE.get_device_type()) while True: diff --git a/selfdrive/version.py b/selfdrive/version.py index 7a2234e3e..2ddfc93c6 100644 --- a/selfdrive/version.py +++ b/selfdrive/version.py @@ -2,6 +2,7 @@ import os import subprocess from typing import List, Optional +from functools import lru_cache from common.basedir import BASEDIR from selfdrive.swaglog import cloudlog @@ -9,9 +10,16 @@ from selfdrive.swaglog import cloudlog TESTED_BRANCHES = ['devel', 'release2-staging', 'release3-staging', 'dashcam-staging', 'release2', 'release3', 'dashcam'] +training_version: bytes = b"0.2.0" +terms_version: bytes = b"2" + + +def cache(user_function, /): + return lru_cache(maxsize=None)(user_function) + def run_cmd(cmd: List[str]) -> str: - return subprocess.check_output(cmd, encoding='utf8').strip() + return subprocess.check_output(cmd, encoding='utf8').strip() def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[str]: @@ -21,19 +29,23 @@ def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[s return default -def get_git_commit(branch: str = "HEAD", default: Optional[str] = None) -> Optional[str]: +@cache +def get_commit(branch: str = "HEAD", default: Optional[str] = None) -> Optional[str]: return run_cmd_default(["git", "rev-parse", branch], default=default) -def get_git_branch(default: Optional[str] = None) -> Optional[str]: +@cache +def get_short_branch(default: Optional[str] = None) -> Optional[str]: return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], default=default) -def get_git_full_branchname(default: Optional[str] = None) -> Optional[str]: +@cache +def get_branch(default: Optional[str] = None) -> Optional[str]: return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], default=default) -def get_git_remote(default: Optional[str] = None) -> Optional[str]: +@cache +def get_origin(default: Optional[str] = None) -> Optional[str]: try: local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"]) tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"]) @@ -42,55 +54,68 @@ def get_git_remote(default: Optional[str] = None) -> Optional[str]: return run_cmd_default(["git", "config", "--get", "remote.origin.url"], default=default) -def get_version(): +@cache +def get_version() -> str: with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "common", "version.h")) as _versionf: version = _versionf.read().split('"')[1] return version -version = get_version() -prebuilt = os.path.exists(os.path.join(BASEDIR, 'prebuilt')) -training_version: bytes = b"0.2.0" -terms_version: bytes = b"2" +@cache +def get_prebuilt() -> bool: + return os.path.exists(os.path.join(BASEDIR, 'prebuilt')) -dirty: bool = True -comma_remote: bool = False -tested_branch: bool = False -origin = get_git_remote() -branch = get_git_full_branchname() -commit = get_git_commit() -if (origin is not None) and (branch is not None): +@cache +def get_comma_remote() -> bool: + origin = get_origin() + if origin is None: + return False + + return origin.startswith('git@github.com:commaai') or origin.startswith('https://github.com/commaai') + + +@cache +def get_tested_branch() -> bool: + return get_short_branch() in TESTED_BRANCHES + + +@cache +def get_dirty() -> bool: + origin = get_origin() + branch = get_branch() + if (origin is None) or (branch is None): + return True + + dirty = False try: - comma_remote = origin.startswith('git@github.com:commaai') or origin.startswith('https://github.com/commaai') - tested_branch = get_git_branch() in TESTED_BRANCHES - - dirty = False - # Actually check dirty files - if not prebuilt: + if not get_prebuilt(): # This is needed otherwise touched files might show up as modified try: subprocess.check_call(["git", "update-index", "--refresh"]) except subprocess.CalledProcessError: pass + dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0) # Log dirty files - if dirty and comma_remote: + if dirty and get_comma_remote(): try: dirty_files = run_cmd(["git", "diff-index", branch, "--"]) - cloudlog.event("dirty comma branch", version=version, dirty=dirty, origin=origin, branch=branch, - dirty_files=dirty_files, commit=commit, origin_commit=get_git_commit(branch)) + cloudlog.event("dirty comma branch", version=get_version(), dirty=dirty, origin=origin, branch=branch, + dirty_files=dirty_files, commit=get_commit(), origin_commit=get_commit(branch)) except subprocess.CalledProcessError: pass - dirty = dirty or (not comma_remote) + dirty = dirty or (not get_comma_remote()) dirty = dirty or ('master' in branch) except subprocess.CalledProcessError: - dirty = True cloudlog.exception("git subprocess failed while checking dirty") + dirty = True + + return dirty if __name__ == "__main__": @@ -100,8 +125,9 @@ if __name__ == "__main__": params.put("TermsVersion", terms_version) params.put("TrainingVersion", training_version) - print("Dirty: %s" % dirty) - print("Version: %s" % version) - print("Remote: %s" % origin) - print("Branch: %s" % branch) - print("Prebuilt: %s" % prebuilt) + print("Dirty: %s" % get_dirty()) + print("Version: %s" % get_version()) + print("Origin: %s" % get_origin()) + print("Branch: %s" % get_branch()) + print("Short branch: %s" % get_short_branch()) + print("Prebuilt: %s" % get_prebuilt())