agnos updater (#2600)
* agnos updater * add manifest * fix path * get manifest from overlay * update manifest * remove merge markers * add streaming decompressor * dont need read all * Unsparsify * Fix output filename * Optimization * cleanup * Small cleanup * Read manifest from merged overlay * Write hash at end of partition * Sync before writing hash * Write bytes in file * add manifest with image sizes * Fix manifest path * File was closed already * Format string * Put raw hash * Read hashes in launch script * update launch script * should be agnos version * fix slot * Make sure we clear the hash * Verify partition size * move updated * Standalone flasher * Don't rely on ordering * Get path * Debug log * Download agnos * Info is enough * update manifest * Remove f * Check downloader return code * Exit on wrong manifest * Fix typos * Set pythonpath before hardware init * move agnos into hardware folder * remove comments * Fix abstractmethod Co-authored-by: Comma Device <device@comma.ai> Co-authored-by: Willem Melching <willem.melching@gmail.com>pull/2004/head
Adeeb Shihadeh
GitHub
parent
8039361567
commit
b276881fcd
|
|
@ -24,6 +24,9 @@ class Spinner():
|
|||
except BrokenPipeError:
|
||||
pass
|
||||
|
||||
def update_progress(self, cur, total):
|
||||
self.update(str(int(100 * cur / total)))
|
||||
|
||||
def close(self):
|
||||
if self.spinner_proc is not None:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
[
|
||||
{
|
||||
"name": "system",
|
||||
"url": "https://commadist.azureedge.net/agnosupdate-staging/system-1e5748c84d9b48bbe599c88da63e46b7ed1c0f1245486cfbcfb05399bd16dbb6.img.xz",
|
||||
"hash": "de6afb8651dc011fed8a883bac191e61d4b58e2e1c84c2717816b64f71afd3b1",
|
||||
"hash_raw": "1e5748c84d9b48bbe599c88da63e46b7ed1c0f1245486cfbcfb05399bd16dbb6",
|
||||
"size": 10737418240,
|
||||
"sparse": true
|
||||
},
|
||||
{
|
||||
"name": "boot",
|
||||
"url": "https://commadist.azureedge.net/agnosupdate-staging/boot-21649cb35935a92776155e7bb23e437e7e7eb0500c082df89f59a6d4a4ed639a.img.xz",
|
||||
"hash": "21649cb35935a92776155e7bb23e437e7e7eb0500c082df89f59a6d4a4ed639a",
|
||||
"hash_raw": "21649cb35935a92776155e7bb23e437e7e7eb0500c082df89f59a6d4a4ed639a",
|
||||
"size": 14678016,
|
||||
"sparse": false
|
||||
}
|
||||
]
|
||||
|
|
@ -8,11 +8,6 @@ source "$BASEDIR/launch_env.sh"
|
|||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||
|
||||
function tici_init {
|
||||
sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu0/governor'
|
||||
sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu4/governor'
|
||||
}
|
||||
|
||||
function two_init {
|
||||
# Wifi scan
|
||||
wpa_cli IFNAME=wlan0 SCAN
|
||||
|
|
@ -110,6 +105,62 @@ function two_init {
|
|||
fi
|
||||
}
|
||||
|
||||
function tici_init {
|
||||
sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu0/governor'
|
||||
sudo su -c 'echo "performance" > /sys/class/devfreq/soc:qcom,memlat-cpu4/governor'
|
||||
|
||||
# set success flag for current boot slot
|
||||
sudo abctl --set_success
|
||||
|
||||
# Check if AGNOS update is required
|
||||
if [ $(< /VERSION) != "$AGNOS_VERSION" ]; then
|
||||
# Get number of slot to switch to
|
||||
CUR_SLOT=$(abctl --boot_slot)
|
||||
if [[ "$CUR_SLOT" == "_a" ]]; then
|
||||
OTHER_SLOT="_b"
|
||||
OTHER_SLOT_NUMBER="1"
|
||||
else
|
||||
OTHER_SLOT="_a"
|
||||
OTHER_SLOT_NUMBER="0"
|
||||
fi
|
||||
echo "Cur slot $CUR_SLOT, target $OTHER_SLOT"
|
||||
|
||||
# Get expected hashes from manifest
|
||||
MANIFEST="$DIR/installer/updater/update_agnos.json"
|
||||
SYSTEM_HASH_EXPECTED=$(jq -r ".[] | select(.name == \"system\") | .hash_raw" $MANIFEST)
|
||||
SYSTEM_SIZE=$(jq -r ".[] | select(.name == \"system\") | .size" $MANIFEST)
|
||||
BOOT_HASH_EXPECTED=$(jq -r ".[] | select(.name == \"boot\") | .hash_raw" $MANIFEST)
|
||||
BOOT_SIZE=$(jq -r ".[] | select(.name == \"boot\") | .size" $MANIFEST)
|
||||
echo "Expected hashes:"
|
||||
echo "System: $SYSTEM_HASH_EXPECTED"
|
||||
echo "Boot: $BOOT_HASH_EXPECTED"
|
||||
|
||||
# Read hashes from alternate partitions, should already be flashed by updated
|
||||
SYSTEM_HASH=$(dd if="/dev/disk/by-partlabel/system$OTHER_SLOT" bs=1 skip="$SYSTEM_SIZE" count=64 2>/dev/null)
|
||||
BOOT_HASH=$(dd if="/dev/disk/by-partlabel/boot$OTHER_SLOT" bs=1 skip="$BOOT_SIZE" count=64 2>/dev/null)
|
||||
echo "Found hashes:"
|
||||
echo "System: $SYSTEM_HASH"
|
||||
echo "Boot: $BOOT_HASH"
|
||||
|
||||
if [[ "$SYSTEM_HASH" == "$SYSTEM_HASH_EXPECTED" && "$BOOT_HASH" == "$BOOT_HASH_EXPECTED" ]]; then
|
||||
echo "Swapping active slot to $OTHER_SLOT_NUMBER"
|
||||
abctl --set_active "$OTHER_SLOT_NUMBER"
|
||||
|
||||
sleep 1
|
||||
sudo reboot
|
||||
else
|
||||
echo "Hash mismatch, downloading agnos"
|
||||
if $DIR/selfdrive/hardware/tici/agnos.py $MANIFEST; then
|
||||
echo "Download done, swapping active slot to $OTHER_SLOT_NUMBER"
|
||||
abctl --set_active "$OTHER_SLOT_NUMBER"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
sudo reboot
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function launch {
|
||||
# Remove orphaned git lock if it exists on boot
|
||||
[ -f "$DIR/.git/index.lock" ] && rm -f $DIR/.git/index.lock
|
||||
|
|
@ -153,17 +204,17 @@ function launch {
|
|||
fi
|
||||
fi
|
||||
|
||||
# comma two init
|
||||
# handle pythonpath
|
||||
ln -sfn $(pwd) /data/pythonpath
|
||||
export PYTHONPATH="$PWD"
|
||||
|
||||
# hardware specific init
|
||||
if [ -f /EON ]; then
|
||||
two_init
|
||||
elif [ -f /TICI ]; then
|
||||
tici_init
|
||||
fi
|
||||
|
||||
# handle pythonpath
|
||||
ln -sfn $(pwd) /data/pythonpath
|
||||
export PYTHONPATH="$PWD"
|
||||
|
||||
# write tmux scrollback to a file
|
||||
tmux capture-pane -pq -S-1000 > /tmp/launch_log
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@ if [ -z "$REQUIRED_NEOS_VERSION" ]; then
|
|||
export REQUIRED_NEOS_VERSION="15-1"
|
||||
fi
|
||||
|
||||
if [ -z "$AGNOS_VERSION" ]; then
|
||||
export AGNOS_VERSION="0.1"
|
||||
fi
|
||||
|
||||
if [ -z "$PASSIVE" ]; then
|
||||
export PASSIVE="1"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -277,6 +277,7 @@ selfdrive/hardware/eon/hardware.py
|
|||
selfdrive/hardware/tici/__init__.py
|
||||
selfdrive/hardware/tici/hardware.py
|
||||
selfdrive/hardware/tici/pins.py
|
||||
selfdrive/hardware/tici/agnos.py
|
||||
selfdrive/hardware/pc/__init__.py
|
||||
selfdrive/hardware/pc/hardware.py
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,10 @@ class HardwareBase:
|
|||
def uninstall(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_os_version(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_sound_card_online(self):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -61,6 +61,10 @@ def getprop(key):
|
|||
|
||||
|
||||
class Android(HardwareBase):
|
||||
def get_os_version(self):
|
||||
with open("/VERSION") as f:
|
||||
return f.read().strip()
|
||||
|
||||
def get_sound_card_online(self):
|
||||
return (os.path.isfile('/proc/asound/card0/state') and
|
||||
open('/proc/asound/card0/state').read().strip() == 'ONLINE')
|
||||
|
|
|
|||
|
|
@ -8,6 +8,9 @@ NetworkStrength = log.ThermalData.NetworkStrength
|
|||
|
||||
|
||||
class Pc(HardwareBase):
|
||||
def get_os_version(self):
|
||||
return None
|
||||
|
||||
def get_sound_card_online(self):
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env python3
|
||||
import json
|
||||
import lzma
|
||||
import hashlib
|
||||
import requests
|
||||
import struct
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
from common.spinner import Spinner
|
||||
|
||||
|
||||
class StreamingDecompressor:
|
||||
def __init__(self, url):
|
||||
self.buf = b""
|
||||
|
||||
self.req = requests.get(url, stream=True, headers={'Accept-Encoding': None})
|
||||
self.it = self.req.iter_content(chunk_size=1024 * 1024)
|
||||
self.decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
|
||||
self.eof = False
|
||||
self.sha256 = hashlib.sha256()
|
||||
|
||||
def read(self, length):
|
||||
while len(self.buf) < length:
|
||||
self.req.raise_for_status()
|
||||
|
||||
try:
|
||||
compressed = next(self.it)
|
||||
except StopIteration:
|
||||
self.eof = True
|
||||
break
|
||||
out = self.decompressor.decompress(compressed)
|
||||
self.buf += out
|
||||
|
||||
result = self.buf[:length]
|
||||
self.buf = self.buf[length:]
|
||||
|
||||
self.sha256.update(result)
|
||||
return result
|
||||
|
||||
|
||||
def unsparsify(f):
|
||||
magic = struct.unpack("I", f.read(4))[0]
|
||||
assert(magic == 0xed26ff3a)
|
||||
|
||||
# Version
|
||||
major = struct.unpack("H", f.read(2))[0]
|
||||
minor = struct.unpack("H", f.read(2))[0]
|
||||
assert(major == 1 and minor == 0)
|
||||
|
||||
# Header sizes
|
||||
_ = struct.unpack("H", f.read(2))[0]
|
||||
_ = struct.unpack("H", f.read(2))[0]
|
||||
|
||||
block_sz = struct.unpack("I", f.read(4))[0]
|
||||
_ = struct.unpack("I", f.read(4))[0]
|
||||
num_chunks = struct.unpack("I", f.read(4))[0]
|
||||
_ = struct.unpack("I", f.read(4))[0]
|
||||
|
||||
for _ in range(num_chunks):
|
||||
chunk_type = struct.unpack("H", f.read(2))[0]
|
||||
_ = struct.unpack("H", f.read(2))[0]
|
||||
out_blocks = struct.unpack("I", f.read(4))[0]
|
||||
_ = struct.unpack("I", f.read(4))[0]
|
||||
|
||||
if chunk_type == 0xcac1: # Raw
|
||||
# TODO: yield in smaller chunks. Yielding only block_sz is too slow. Largest observed data chunk is 252 MB.
|
||||
yield f.read(out_blocks * block_sz)
|
||||
elif chunk_type == 0xcac2: # Fill
|
||||
filler = f.read(4) * (block_sz // 4)
|
||||
for _ in range(out_blocks):
|
||||
yield filler
|
||||
elif chunk_type == 0xcac3: # Don't care
|
||||
yield b""
|
||||
else:
|
||||
raise Exception("Unhandled sparse chunk type")
|
||||
|
||||
|
||||
def flash_agnos_update(manifest_path, cloudlog, spinner=None):
|
||||
update = json.load(open(manifest_path))
|
||||
|
||||
current_slot = subprocess.check_output(["abctl", "--boot_slot"], encoding='utf-8').strip()
|
||||
target_slot = "_b" if current_slot == "_a" else "_a"
|
||||
target_slot_number = "0" if target_slot == "_a" else "1"
|
||||
|
||||
cloudlog.info(f"Current slot {current_slot}, target slot {target_slot}")
|
||||
|
||||
# set target slot as unbootable
|
||||
os.system(f"abctl --set_unbootable {target_slot_number}")
|
||||
|
||||
for partition in update:
|
||||
cloudlog.info(f"Downloading and writing {partition['name']}")
|
||||
|
||||
downloader = StreamingDecompressor(partition['url'])
|
||||
with open(f"/dev/disk/by-partlabel/{partition['name']}{target_slot}", 'wb') as out:
|
||||
partition_size = partition['size']
|
||||
# Clear hash before flashing
|
||||
out.seek(partition_size)
|
||||
out.write(b"\x00" * 64)
|
||||
out.seek(0)
|
||||
os.sync()
|
||||
|
||||
# Flash partition
|
||||
if partition['sparse']:
|
||||
raw_hash = hashlib.sha256()
|
||||
for chunk in unsparsify(downloader):
|
||||
raw_hash.update(chunk)
|
||||
out.write(chunk)
|
||||
|
||||
if spinner is not None:
|
||||
spinner.update_progress(out.tell(), partition_size)
|
||||
|
||||
if raw_hash.hexdigest().lower() != partition['hash_raw'].lower():
|
||||
raise Exception(f"Unsparse hash mismatch '{raw_hash.hexdigest().lower()}'")
|
||||
else:
|
||||
while not downloader.eof:
|
||||
out.write(downloader.read(1024 * 1024))
|
||||
|
||||
if spinner is not None:
|
||||
spinner.update_progress(out.tell(), partition_size)
|
||||
|
||||
if downloader.sha256.hexdigest().lower() != partition['hash'].lower():
|
||||
raise Exception("Uncompressed hash mismatch")
|
||||
|
||||
if out.tell() != partition['size']:
|
||||
raise Exception("Uncompressed size mismatch")
|
||||
|
||||
# Write hash after successfull flash
|
||||
os.sync()
|
||||
out.write(partition['hash_raw'].lower().encode())
|
||||
|
||||
cloudlog.info(f"AGNOS ready on slot {target_slot}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import logging
|
||||
import time
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("Usage: ./agnos.py <manifest.json>")
|
||||
exit(1)
|
||||
|
||||
spinner = Spinner()
|
||||
spinner.update("Updating AGNOS")
|
||||
time.sleep(5)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
flash_agnos_update(sys.argv[1], logging, spinner)
|
||||
|
|
@ -32,6 +32,10 @@ class Tici(HardwareBase):
|
|||
self.nm = self.bus.get_object(NM, '/org/freedesktop/NetworkManager')
|
||||
self.mm = self.bus.get_object(MM, '/org/freedesktop/ModemManager1')
|
||||
|
||||
def get_os_version(self):
|
||||
with open("/VERSION") as f:
|
||||
return f.read().strip()
|
||||
|
||||
def get_sound_card_online(self):
|
||||
return True
|
||||
|
||||
|
|
|
|||
|
|
@ -36,14 +36,14 @@ from typing import List, Tuple, Optional
|
|||
|
||||
from common.basedir import BASEDIR
|
||||
from common.params import Params
|
||||
from selfdrive.hardware import EON, TICI
|
||||
from selfdrive.hardware import EON, TICI, HARDWARE
|
||||
from selfdrive.swaglog import cloudlog
|
||||
from selfdrive.controls.lib.alertmanager import set_offroad_alert
|
||||
from selfdrive.hardware.tici.agnos import flash_agnos_update
|
||||
|
||||
LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
|
||||
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
|
||||
|
||||
NEOS_VERSION = os.getenv("UPDATER_NEOS_VERSION", "/VERSION")
|
||||
NEOSUPDATE_DIR = os.getenv("UPDATER_NEOSUPDATE_DIR", "/data/neoupdate")
|
||||
|
||||
OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
|
||||
|
|
@ -214,10 +214,23 @@ def finalize_update() -> None:
|
|||
cloudlog.info("done finalizing overlay")
|
||||
|
||||
|
||||
def handle_neos_update(wait_helper: WaitTimeHelper) -> None:
|
||||
with open(NEOS_VERSION, "r") as f:
|
||||
cur_neos = f.read().strip()
|
||||
def handle_agnos_update(wait_helper):
|
||||
cur_version = HARDWARE.get_os_version()
|
||||
updated_version = run(["bash", "-c", r"unset AGNOS_VERSION && source launch_env.sh && \
|
||||
echo -n $AGNOS_VERSION"], OVERLAY_MERGED).strip()
|
||||
|
||||
cloudlog.info(f"AGNOS version check: {cur_version} vs {updated_version}")
|
||||
if cur_version == updated_version:
|
||||
return
|
||||
|
||||
cloudlog.info(f"Beginning background installation for AGNOS {updated_version}")
|
||||
|
||||
manifest_path = os.path.join(OVERLAY_MERGED, "installer/updater/update_agnos.json")
|
||||
flash_agnos_update(manifest_path, cloudlog)
|
||||
|
||||
|
||||
def handle_neos_update(wait_helper: WaitTimeHelper) -> None:
|
||||
cur_neos = HARDWARE.get_os_version()
|
||||
updated_neos = run(["bash", "-c", r"unset REQUIRED_NEOS_VERSION && source launch_env.sh && \
|
||||
echo -n $REQUIRED_NEOS_VERSION"], OVERLAY_MERGED).strip()
|
||||
|
||||
|
|
@ -294,6 +307,8 @@ def fetch_update(wait_helper: WaitTimeHelper) -> bool:
|
|||
|
||||
if EON:
|
||||
handle_neos_update(wait_helper)
|
||||
elif TICI:
|
||||
handle_agnos_update(wait_helper)
|
||||
|
||||
# Create the finalized, ready-to-swap update
|
||||
finalize_update()
|
||||
|
|
@ -392,5 +407,6 @@ def main():
|
|||
|
||||
dismount_overlay()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue