run onroad tests on release build (#22700)
* check startup aelrt * release decorator * run in jenkins * run onroad tests * no push * fix release2 build * fix path * no overwrite * single release build script * make files_eon non empty * files * run in source * add that for now * print * ignore initialzing * print * fetch * run tests last * check alert text * revert some stuff * fixup jenkinsfile Co-authored-by: Comma Device <device@comma.ai>pull/22711/head
parent
5b331fd6f5
commit
7feae28705
|
@ -70,7 +70,7 @@ pipeline {
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
phone_steps("eon-build", [
|
phone_steps("eon-build", [
|
||||||
["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"],
|
["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release.sh"],
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -87,7 +87,7 @@ pipeline {
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
phone_steps("tici", [
|
phone_steps("tici", [
|
||||||
["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release3.sh"],
|
["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"],
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -9,7 +9,17 @@ cd $DIR
|
||||||
BUILD_DIR=/data/openpilot
|
BUILD_DIR=/data/openpilot
|
||||||
SOURCE_DIR="$(git rev-parse --show-toplevel)"
|
SOURCE_DIR="$(git rev-parse --show-toplevel)"
|
||||||
|
|
||||||
BRANCH=release3-staging
|
if [ -f /TICI ]; then
|
||||||
|
FILES_SRC="release/files_tici"
|
||||||
|
RELEASE_BRANCH=release3-staging
|
||||||
|
DASHBAM_BRANCH=dashcam3-staging
|
||||||
|
elif [ -f /EON ]; then
|
||||||
|
FILES_SRC="release/files_eon"
|
||||||
|
RELEASE_BRANCH=release2-staging
|
||||||
|
DASHBAM_BRANCH=dashcam-staging
|
||||||
|
else
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
# set git identity
|
# set git identity
|
||||||
source $DIR/identity.sh
|
source $DIR/identity.sh
|
||||||
|
@ -20,13 +30,14 @@ mkdir -p $BUILD_DIR
|
||||||
cd $BUILD_DIR
|
cd $BUILD_DIR
|
||||||
git init
|
git init
|
||||||
git remote add origin git@github.com:commaai/openpilot.git
|
git remote add origin git@github.com:commaai/openpilot.git
|
||||||
git checkout -f -B $BRANCH
|
git fetch origin $RELEASE_BRANCH
|
||||||
|
git checkout --orphan $RELEASE_BRANCH
|
||||||
|
|
||||||
# do the files copy
|
# do the files copy
|
||||||
echo "[-] copying files T=$SECONDS"
|
echo "[-] copying files T=$SECONDS"
|
||||||
cd $SOURCE_DIR
|
cd $SOURCE_DIR
|
||||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/
|
cp -pR --parents $(cat release/files_common) $BUILD_DIR/
|
||||||
cp -pR --parents $(cat release/files_tici) $BUILD_DIR/
|
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
|
||||||
|
|
||||||
# in the directory
|
# in the directory
|
||||||
cd $BUILD_DIR
|
cd $BUILD_DIR
|
||||||
|
@ -39,6 +50,7 @@ echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h
|
||||||
echo "[-] committing version $VERSION T=$SECONDS"
|
echo "[-] committing version $VERSION T=$SECONDS"
|
||||||
git add -f .
|
git add -f .
|
||||||
git commit -a -m "openpilot v$VERSION release"
|
git commit -a -m "openpilot v$VERSION release"
|
||||||
|
git branch --set-upstream-to=origin/$RELEASE_BRANCH
|
||||||
|
|
||||||
# Build panda firmware
|
# Build panda firmware
|
||||||
pushd panda/
|
pushd panda/
|
||||||
|
@ -50,15 +62,20 @@ popd
|
||||||
export PYTHONPATH="$BUILD_DIR"
|
export PYTHONPATH="$BUILD_DIR"
|
||||||
scons -j$(nproc)
|
scons -j$(nproc)
|
||||||
|
|
||||||
# Run tests
|
# Ensure no submodules in release
|
||||||
#python selfdrive/manager/test/test_manager.py
|
if test "$(git submodule--helper list | wc -l)" -gt "0"; then
|
||||||
selfdrive/car/tests/test_car_interfaces.py
|
echo "submodules found:"
|
||||||
|
git submodule--helper list
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
git submodule status
|
||||||
|
|
||||||
# Cleanup
|
# Cleanup
|
||||||
find . -name '*.a' -delete
|
find . -name '*.a' -delete
|
||||||
find . -name '*.o' -delete
|
find . -name '*.o' -delete
|
||||||
find . -name '*.os' -delete
|
find . -name '*.os' -delete
|
||||||
find . -name '*.pyc' -delete
|
find . -name '*.pyc' -delete
|
||||||
|
find . -name 'moc_*' -delete
|
||||||
find . -name '__pycache__' -delete
|
find . -name '__pycache__' -delete
|
||||||
rm -rf panda/board panda/certs panda/crypto
|
rm -rf panda/board panda/certs panda/crypto
|
||||||
rm -rf .sconsign.dblite Jenkinsfile release/
|
rm -rf .sconsign.dblite Jenkinsfile release/
|
||||||
|
@ -78,15 +95,24 @@ touch prebuilt
|
||||||
git add -f .
|
git add -f .
|
||||||
git commit --amend -m "openpilot v$VERSION"
|
git commit --amend -m "openpilot v$VERSION"
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
TEST_FILES="tools/"
|
||||||
|
cd $SOURCE_DIR
|
||||||
|
cp -pR -n --parents $TEST_FILES $BUILD_DIR/
|
||||||
|
cd $BUILD_DIR
|
||||||
|
RELEASE=1 selfdrive/test/test_onroad.py
|
||||||
|
#selfdrive/manager/test/test_manager.py
|
||||||
|
selfdrive/car/tests/test_car_interfaces.py
|
||||||
|
rm -rf $TEST_FILES
|
||||||
|
|
||||||
if [ ! -z "$PUSH" ]; then
|
if [ ! -z "$PUSH" ]; then
|
||||||
echo "[-] pushing T=$SECONDS"
|
echo "[-] pushing T=$SECONDS"
|
||||||
git remote set-url origin git@github.com:commaai/openpilot.git
|
git push -f origin $RELEASE_BRANCH
|
||||||
git push -f origin $BRANCH
|
|
||||||
|
|
||||||
# Create dashcam
|
# Create dashcam
|
||||||
git rm selfdrive/car/*/carcontroller.py
|
git rm selfdrive/car/*/carcontroller.py
|
||||||
git commit -m "create dashcam release from release"
|
git commit -m "create dashcam release from release"
|
||||||
git push -f origin $BRANCH:dashcam3-staging
|
git push -f origin $RELEASE_BRANCH:$DASHCAM_BRANCH
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "[-] done T=$SECONDS"
|
echo "[-] done T=$SECONDS"
|
|
@ -1,104 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
export GIT_COMMITTER_NAME="Vehicle Researcher"
|
|
||||||
export GIT_COMMITTER_EMAIL="user@comma.ai"
|
|
||||||
export GIT_AUTHOR_NAME="Vehicle Researcher"
|
|
||||||
export GIT_AUTHOR_EMAIL="user@comma.ai"
|
|
||||||
|
|
||||||
export GIT_SSH_COMMAND="ssh -i /data/gitkey"
|
|
||||||
|
|
||||||
# set CLEAN to build outside of CI
|
|
||||||
if [ ! -z "$CLEAN" ]; then
|
|
||||||
# Create folders
|
|
||||||
rm -rf /data/openpilot
|
|
||||||
mkdir -p /data/openpilot
|
|
||||||
cd /data/openpilot
|
|
||||||
|
|
||||||
# Create git repo
|
|
||||||
git init
|
|
||||||
git remote add origin git@github.com:commaai/openpilot.git
|
|
||||||
git fetch origin devel-staging
|
|
||||||
else
|
|
||||||
cd /data/openpilot
|
|
||||||
git clean -xdf
|
|
||||||
git branch -D release2-staging || true
|
|
||||||
fi
|
|
||||||
|
|
||||||
git fetch origin release2-staging
|
|
||||||
git fetch origin dashcam-staging
|
|
||||||
|
|
||||||
# Create release2 with no history
|
|
||||||
if [ ! -z "$CLEAN" ]; then
|
|
||||||
git checkout --orphan release2-staging origin/devel-staging
|
|
||||||
else
|
|
||||||
git checkout --orphan release2-staging
|
|
||||||
fi
|
|
||||||
|
|
||||||
VERSION=$(cat selfdrive/common/version.h | awk -F[\"-] '{print $2}')
|
|
||||||
echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h
|
|
||||||
|
|
||||||
git commit -m "openpilot v$VERSION"
|
|
||||||
|
|
||||||
# Build signed panda firmware
|
|
||||||
pushd panda/
|
|
||||||
CERT=/tmp/pandaextra/certs/release RELEASE=1 scons -u .
|
|
||||||
mv board/obj/panda.bin.signed /tmp/panda.bin.signed
|
|
||||||
popd
|
|
||||||
|
|
||||||
# Build stuff
|
|
||||||
ln -sfn /data/openpilot /data/pythonpath
|
|
||||||
export PYTHONPATH="/data/openpilot:/data/openpilot/pyextra"
|
|
||||||
scons -j3
|
|
||||||
|
|
||||||
# Run tests
|
|
||||||
python selfdrive/manager/test/test_manager.py
|
|
||||||
selfdrive/car/tests/test_car_interfaces.py
|
|
||||||
|
|
||||||
# Ensure no submodules in release
|
|
||||||
if test "$(git submodule--helper list | wc -l)" -gt "0"; then
|
|
||||||
echo "submodules found:"
|
|
||||||
git submodule--helper list
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
git submodule status
|
|
||||||
|
|
||||||
# Cleanup
|
|
||||||
find . -name '*.a' -delete
|
|
||||||
find . -name '*.o' -delete
|
|
||||||
find . -name '*.os' -delete
|
|
||||||
find . -name '*.pyc' -delete
|
|
||||||
find . -name '__pycache__' -delete
|
|
||||||
rm -rf panda/board panda/certs panda/crypto
|
|
||||||
rm -rf .sconsign.dblite Jenkinsfile release/
|
|
||||||
rm models/supercombo.dlc
|
|
||||||
|
|
||||||
# Move back signed panda fw
|
|
||||||
mkdir -p panda/board/obj
|
|
||||||
mv /tmp/panda.bin.signed panda/board/obj/panda.bin.signed
|
|
||||||
|
|
||||||
# Restore third_party
|
|
||||||
git checkout third_party/
|
|
||||||
|
|
||||||
# Mark as prebuilt release
|
|
||||||
touch prebuilt
|
|
||||||
|
|
||||||
# Add built files to git
|
|
||||||
git add -f .
|
|
||||||
git commit --amend -m "openpilot v$VERSION"
|
|
||||||
|
|
||||||
# Print committed files that are normally gitignored
|
|
||||||
#git status --ignored
|
|
||||||
|
|
||||||
if [ ! -z "$PUSH" ]; then
|
|
||||||
git remote set-url origin git@github.com:commaai/openpilot.git
|
|
||||||
|
|
||||||
# Push to release2-staging
|
|
||||||
git push -f origin release2-staging
|
|
||||||
|
|
||||||
# Create dashcam release
|
|
||||||
git rm selfdrive/car/*/carcontroller.py
|
|
||||||
|
|
||||||
git commit -m "create dashcam release from release2"
|
|
||||||
git push -f origin release2-staging:dashcam-staging
|
|
||||||
fi
|
|
|
@ -0,0 +1 @@
|
||||||
|
README.md
|
|
@ -1,3 +1,4 @@
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
|
|
||||||
|
@ -24,6 +25,13 @@ def phone_only(f):
|
||||||
f(self, *args, **kwargs)
|
f(self, *args, **kwargs)
|
||||||
return wrap
|
return wrap
|
||||||
|
|
||||||
|
def release_only(f):
|
||||||
|
@wraps(f)
|
||||||
|
def wrap(self, *args, **kwargs):
|
||||||
|
if "RELEASE" not in os.environ:
|
||||||
|
self.skipTest("This test is only for release branches")
|
||||||
|
f(self, *args, **kwargs)
|
||||||
|
return wrap
|
||||||
|
|
||||||
def with_processes(processes, init_time=0, ignore_stopped=None):
|
def with_processes(processes, init_time=0, ignore_stopped=None):
|
||||||
ignore_stopped = [] if ignore_stopped is None else ignore_stopped
|
ignore_stopped = [] if ignore_stopped is None else ignore_stopped
|
||||||
|
|
|
@ -8,14 +8,16 @@ import unittest
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from cereal import car
|
||||||
import cereal.messaging as messaging
|
import cereal.messaging as messaging
|
||||||
from cereal.services import service_list
|
from cereal.services import service_list
|
||||||
from common.basedir import BASEDIR
|
from common.basedir import BASEDIR
|
||||||
from common.timeout import Timeout
|
from common.timeout import Timeout
|
||||||
from common.params import Params
|
from common.params import Params
|
||||||
|
from selfdrive.controls.lib.events import EVENTS, ET
|
||||||
from selfdrive.hardware import EON, TICI
|
from selfdrive.hardware import EON, TICI
|
||||||
from selfdrive.loggerd.config import ROOT
|
from selfdrive.loggerd.config import ROOT
|
||||||
from selfdrive.test.helpers import set_params_enabled
|
from selfdrive.test.helpers import set_params_enabled, release_only
|
||||||
from tools.lib.logreader import LogReader
|
from tools.lib.logreader import LogReader
|
||||||
|
|
||||||
# Baseline CPU usage by process
|
# Baseline CPU usage by process
|
||||||
|
@ -56,8 +58,9 @@ if TICI:
|
||||||
"./camerad": 31.0,
|
"./camerad": 31.0,
|
||||||
"./_ui": 21.0,
|
"./_ui": 21.0,
|
||||||
"selfdrive.controls.plannerd": 11.7,
|
"selfdrive.controls.plannerd": 11.7,
|
||||||
"selfdrive.locationd.paramsd": 5.0,
|
|
||||||
"./_dmonitoringmodeld": 10.0,
|
"./_dmonitoringmodeld": 10.0,
|
||||||
|
"selfdrive.locationd.paramsd": 5.0,
|
||||||
|
"selfdrive.controls.radard": 3.6,
|
||||||
"selfdrive.thermald.thermald": 1.5,
|
"selfdrive.thermald.thermald": 1.5,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -128,7 +131,7 @@ class TestOnroad(unittest.TestCase):
|
||||||
if "DEBUG" in os.environ:
|
if "DEBUG" in os.environ:
|
||||||
segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.bz2")), Path(ROOT).iterdir())
|
segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.bz2")), Path(ROOT).iterdir())
|
||||||
segs = sorted(segs, key=lambda x: x.stat().st_mtime)
|
segs = sorted(segs, key=lambda x: x.stat().st_mtime)
|
||||||
cls.lr = list(LogReader(os.path.join(segs[-2], "rlog.bz2")))
|
cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2")))
|
||||||
return
|
return
|
||||||
|
|
||||||
os.environ['SKIP_FW_QUERY'] = "1"
|
os.environ['SKIP_FW_QUERY'] = "1"
|
||||||
|
@ -171,6 +174,9 @@ class TestOnroad(unittest.TestCase):
|
||||||
if proc.wait(60) is None:
|
if proc.wait(60) is None:
|
||||||
proc.kill()
|
proc.kill()
|
||||||
|
|
||||||
|
cls.lrs = [list(LogReader(os.path.join(str(s), "rlog.bz2"))) for s in cls.segments]
|
||||||
|
|
||||||
|
# use the second segment by default as it's the first full segment
|
||||||
cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2")))
|
cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2")))
|
||||||
|
|
||||||
def test_cloudlog_size(self):
|
def test_cloudlog_size(self):
|
||||||
|
@ -222,5 +228,17 @@ class TestOnroad(unittest.TestCase):
|
||||||
print(f" {np.max(np.absolute([np.max(ts)/dt, np.min(ts)/dt]))} {np.std(ts)/dt}")
|
print(f" {np.max(np.absolute([np.max(ts)/dt, np.min(ts)/dt]))} {np.std(ts)/dt}")
|
||||||
print("="*67)
|
print("="*67)
|
||||||
|
|
||||||
|
@release_only
|
||||||
|
def test_startup(self):
|
||||||
|
startup_alert = None
|
||||||
|
for msg in self.lrs[0]:
|
||||||
|
# can't use carEvents because the first msg can be dropped while loggerd is starting up
|
||||||
|
if msg.which() == "controlsState":
|
||||||
|
startup_alert = msg.controlsState.alertText1
|
||||||
|
break
|
||||||
|
expected = EVENTS[car.CarEvent.EventName.startup][ET.PERMANENT].alert_text_1
|
||||||
|
self.assertEqual(startup_alert, expected, "wrong startup alert")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
Loading…
Reference in New Issue