50 lines
1.9 KiB
Python
Executable File
50 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
import shutil
|
|
from azure.storage.blob import BlockBlobService
|
|
|
|
from selfdrive.test.test_car_models import routes as test_car_models_routes, non_public_routes
|
|
from selfdrive.test.process_replay.test_processes import segments as replay_segments
|
|
from xx.chffr.lib import azureutil
|
|
from xx.chffr.lib.storage import upload_dir_serial, download_dir_tpe, key_prefix_exists
|
|
from xx.chffr.lib.storage import _DATA_ACCOUNT_PRODUCTION, _DATA_ACCOUNT_CI, _DATA_BUCKET_PRODUCTION, _DATA_BUCKET_CI
|
|
|
|
sas_token = os.getenv("TOKEN", None)
|
|
if sas_token is None:
|
|
sas_token = subprocess.check_output("az storage container generate-sas --account-name commadataci --name openpilotci --https-only --permissions lrw --expiry $(date -u '+%Y-%m-%dT%H:%M:%SZ' -d '+1 hour') --auth-mode login --as-user --output tsv", shell=True).decode().strip("\n")
|
|
service = BlockBlobService(account_name=_DATA_ACCOUNT_CI, sas_token=sas_token)
|
|
|
|
def sync_to_ci_public(service, route):
|
|
key_prefix = route.replace('|', '/')
|
|
|
|
if next(azureutil.list_all_blobs(service, "openpilotci", prefix=key_prefix), None) is not None:
|
|
return
|
|
|
|
print("uploading", route)
|
|
|
|
tmpdir = tempfile.mkdtemp()
|
|
try:
|
|
print(f"download_dir_tpe({_DATA_ACCOUNT_PRODUCTION}, {_DATA_BUCKET_PRODUCTION}, {key_prefix}, {tmpdir})")
|
|
|
|
# production -> openpilotci
|
|
download_dir_tpe(_DATA_ACCOUNT_PRODUCTION, _DATA_BUCKET_PRODUCTION, tmpdir, key_prefix)
|
|
|
|
# commadataci -> openpilotci
|
|
#download_dir_tpe(_DATA_ACCOUNT_CI, _DATA_BUCKET_CI, tmpdir, key_prefix)
|
|
|
|
upload_dir_serial(_DATA_ACCOUNT_CI, "openpilotci", tmpdir, key_prefix)
|
|
finally:
|
|
shutil.rmtree(tmpdir)
|
|
|
|
# sync process replay routes
|
|
for s in replay_segments:
|
|
route_name, _ = s.rsplit('--', 1)
|
|
sync_to_ci_public(service, route_name)
|
|
|
|
# sync test_car_models routes
|
|
for r in test_car_models_routes:
|
|
if r not in non_public_routes:
|
|
sync_to_ci_public(service, r)
|