On-device CI framework (#1784)
* let's see if this works * fix build_release actions job * does jenkins like this config * separate jenkinsfile for release build * fix devel build * devel build should work * always pass that for now * run modeld replay * release2 build will be a separate PR * pass env to phone shell * force checkout * run on real jenkins eons * add timeout * rsync * more timeout * trailing slash * fix branch detection * debug * not sure why paramiko doesn't pass it through * newline * CI_PUSH * still not passing it * test branch * should be good nowalbatross
parent
d417481e2b
commit
32f03ec8a5
|
@ -4,38 +4,42 @@ pipeline {
|
||||||
image 'python:3.7.3'
|
image 'python:3.7.3'
|
||||||
args '--user=root'
|
args '--user=root'
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
environment {
|
environment {
|
||||||
COMMA_JWT = credentials('athena-test-jwt')
|
COMMA_JWT = credentials('athena-test-jwt')
|
||||||
|
CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
|
||||||
}
|
}
|
||||||
|
|
||||||
stages {
|
stages {
|
||||||
stage('Device Tests') {
|
stage('On-device Tests') {
|
||||||
parallel {
|
parallel {
|
||||||
stage('Build/Test') {
|
|
||||||
|
stage('Build') {
|
||||||
steps {
|
steps {
|
||||||
lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){
|
lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
|
||||||
timeout(time: 30, unit: 'MINUTES') {
|
timeout(time: 30, unit: 'MINUTES') {
|
||||||
dir(path: 'release') {
|
|
||||||
sh 'pip install paramiko'
|
|
||||||
sh 'python remote_build.py'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stage('Replay Tests') {
|
|
||||||
steps {
|
|
||||||
lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_name', quantity: 1){
|
|
||||||
timeout(time: 45, unit: 'MINUTES') {
|
|
||||||
dir(path: 'selfdrive/test') {
|
dir(path: 'selfdrive/test') {
|
||||||
sh 'pip install paramiko'
|
sh 'pip install paramiko'
|
||||||
sh 'python phone_ci.py'
|
sh 'python phone_ci.py "cd release && ./build_devel.sh"'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
stage('Replay Tests') {
|
||||||
|
steps {
|
||||||
|
lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
|
||||||
|
timeout(time: 60, unit: 'MINUTES') {
|
||||||
|
dir(path: 'selfdrive/test') {
|
||||||
|
sh 'pip install paramiko'
|
||||||
|
sh 'python phone_ci.py "cd selfdrive/test/process_replay && ./camera_replay.py"'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -161,7 +161,10 @@ env = Environment(
|
||||||
)
|
)
|
||||||
|
|
||||||
if os.environ.get('SCONS_CACHE'):
|
if os.environ.get('SCONS_CACHE'):
|
||||||
CacheDir('/tmp/scons_cache')
|
if QCOM_REPLAY:
|
||||||
|
CacheDir('/tmp/scons_cache_qcom_replay')
|
||||||
|
else:
|
||||||
|
CacheDir('/tmp/scons_cache')
|
||||||
|
|
||||||
node_interval = 5
|
node_interval = 5
|
||||||
node_count = 0
|
node_count = 0
|
||||||
|
|
|
@ -37,7 +37,7 @@ echo "[-] bringing master-ci and devel in sync T=$SECONDS"
|
||||||
git fetch origin master-ci
|
git fetch origin master-ci
|
||||||
git fetch origin devel
|
git fetch origin devel
|
||||||
|
|
||||||
git checkout --track origin/master-ci || true
|
git checkout -f --track origin/master-ci
|
||||||
git reset --hard master-ci
|
git reset --hard master-ci
|
||||||
git checkout master-ci
|
git checkout master-ci
|
||||||
git reset --hard origin/devel
|
git reset --hard origin/devel
|
||||||
|
@ -93,9 +93,9 @@ pushd panda/board/pedal
|
||||||
make obj/comma.bin
|
make obj/comma.bin
|
||||||
popd
|
popd
|
||||||
|
|
||||||
if [ ! -z "$PUSH" ]; then
|
if [ ! -z "$CI_PUSH" ]; then
|
||||||
echo "[-] Pushing to $PUSH T=$SECONDS"
|
echo "[-] Pushing to $CI_PUSH T=$SECONDS"
|
||||||
git push -f origin master-ci:$PUSH
|
git push -f origin master-ci:$CI_PUSH
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "[-] done pushing T=$SECONDS"
|
echo "[-] done pushing T=$SECONDS"
|
||||||
|
|
|
@ -1,16 +1,16 @@
|
||||||
README.md
|
|
||||||
SAFETY.md
|
|
||||||
|
|
||||||
.gitignore
|
.gitignore
|
||||||
LICENSE
|
LICENSE
|
||||||
launch_chffrplus.sh
|
launch_chffrplus.sh
|
||||||
launch_openpilot.sh
|
launch_openpilot.sh
|
||||||
|
|
||||||
CONTRIBUTING.md
|
Jenkinsfile
|
||||||
RELEASES.md
|
|
||||||
|
|
||||||
SConstruct
|
SConstruct
|
||||||
|
|
||||||
|
CONTRIBUTING.md
|
||||||
|
README.md
|
||||||
|
RELEASES.md
|
||||||
|
SAFETY.md
|
||||||
|
|
||||||
apk/ai.comma*.apk
|
apk/ai.comma*.apk
|
||||||
|
|
||||||
common/.gitignore
|
common/.gitignore
|
||||||
|
|
|
@ -1,73 +0,0 @@
|
||||||
#!/usr/bin/env python2
|
|
||||||
import paramiko # pylint: disable=import-error
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
import socket
|
|
||||||
|
|
||||||
|
|
||||||
def start_build(name):
|
|
||||||
ssh = paramiko.SSHClient()
|
|
||||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
||||||
|
|
||||||
key_file = open(os.path.join(os.path.dirname(__file__), "../tools/ssh/key/id_rsa"))
|
|
||||||
key = paramiko.RSAKey.from_private_key(key_file)
|
|
||||||
|
|
||||||
print("SSH to phone {}".format(name))
|
|
||||||
|
|
||||||
# Try connecting for one minute
|
|
||||||
t_start = time.time()
|
|
||||||
while True:
|
|
||||||
try:
|
|
||||||
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
|
|
||||||
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
|
|
||||||
print("Connection failed")
|
|
||||||
if time.time() - t_start > 60:
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
break
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
conn = ssh.invoke_shell()
|
|
||||||
branch = os.environ['GIT_BRANCH']
|
|
||||||
commit = os.environ.get('GIT_COMMIT', branch)
|
|
||||||
|
|
||||||
conn.send('uname -a\n')
|
|
||||||
|
|
||||||
conn.send('cd /data/openpilot_source\n')
|
|
||||||
conn.send("git reset --hard\n")
|
|
||||||
conn.send("git fetch origin\n")
|
|
||||||
conn.send("git checkout %s\n" % commit)
|
|
||||||
conn.send("git clean -xdf\n")
|
|
||||||
conn.send("git submodule update --init\n")
|
|
||||||
conn.send("git submodule foreach --recursive git reset --hard\n")
|
|
||||||
conn.send("git submodule foreach --recursive git clean -xdf\n")
|
|
||||||
conn.send("echo \"git took $SECONDS seconds\"\n")
|
|
||||||
|
|
||||||
push = "PUSH=master-ci" if branch == "master" else ""
|
|
||||||
|
|
||||||
conn.send("%s /data/openpilot_source/release/build_devel.sh\n" % push)
|
|
||||||
conn.send('echo "RESULT:" $?\n')
|
|
||||||
conn.send("exit\n")
|
|
||||||
return conn
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
eon_name = os.environ.get('eon_name', None)
|
|
||||||
|
|
||||||
conn = start_build(eon_name)
|
|
||||||
|
|
||||||
dat = b""
|
|
||||||
|
|
||||||
while True:
|
|
||||||
recvd = conn.recv(4096)
|
|
||||||
if len(recvd) == 0:
|
|
||||||
break
|
|
||||||
|
|
||||||
dat += recvd
|
|
||||||
sys.stdout.buffer.write(recvd)
|
|
||||||
sys.stdout.flush()
|
|
||||||
|
|
||||||
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
|
|
||||||
sys.exit(int(returns[0]))
|
|
|
@ -6,22 +6,29 @@ import re
|
||||||
import time
|
import time
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
TEST_DIR = "/data/openpilotci"
|
|
||||||
|
|
||||||
def run_test(name, test_func):
|
SOURCE_DIR = "/data/openpilot_source/"
|
||||||
|
TEST_DIR = "/data/openpilot/"
|
||||||
|
|
||||||
|
def run_on_phone(test_cmd):
|
||||||
|
|
||||||
|
eon_ip = os.environ.get('eon_ip', None)
|
||||||
|
if eon_ip is None:
|
||||||
|
raise Exception("'eon_ip' not set")
|
||||||
|
|
||||||
ssh = paramiko.SSHClient()
|
ssh = paramiko.SSHClient()
|
||||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
|
||||||
key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
|
key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
|
||||||
key = paramiko.RSAKey.from_private_key(key_file)
|
key = paramiko.RSAKey.from_private_key(key_file)
|
||||||
|
|
||||||
print("SSH to phone {}".format(name))
|
print("SSH to phone at {}".format(eon_ip))
|
||||||
|
|
||||||
# Try connecting for one minute
|
# try connecting for one minute
|
||||||
t_start = time.time()
|
t_start = time.time()
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
|
ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
|
||||||
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
|
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
|
||||||
print("Connection failed")
|
print("Connection failed")
|
||||||
if time.time() - t_start > 60:
|
if time.time() - t_start > 60:
|
||||||
|
@ -30,40 +37,47 @@ def run_test(name, test_func):
|
||||||
break
|
break
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
|
|
||||||
conn = ssh.invoke_shell()
|
|
||||||
branch = os.environ['GIT_BRANCH']
|
branch = os.environ['GIT_BRANCH']
|
||||||
commit = os.environ.get('GIT_COMMIT', branch)
|
commit = os.environ.get('GIT_COMMIT', branch)
|
||||||
|
|
||||||
conn.send("uname -a\n")
|
conn = ssh.invoke_shell()
|
||||||
|
|
||||||
conn.send(f"cd {TEST_DIR}\n")
|
# pass in all environment variables prefixed with 'CI_'
|
||||||
|
for k, v in os.environ.items():
|
||||||
|
if k.startswith("CI_"):
|
||||||
|
conn.send(f"export {k}='{v}'\n")
|
||||||
|
conn.send("export CI=1\n")
|
||||||
|
|
||||||
|
# set up environment
|
||||||
|
conn.send(f"cd {SOURCE_DIR}\n")
|
||||||
conn.send("git reset --hard\n")
|
conn.send("git reset --hard\n")
|
||||||
conn.send("git fetch origin\n")
|
conn.send("git fetch origin\n")
|
||||||
conn.send("git checkout %s\n" % commit)
|
conn.send(f"git checkout {commit}\n")
|
||||||
conn.send("git clean -xdf\n")
|
conn.send("git clean -xdf\n")
|
||||||
conn.send("git submodule update --init\n")
|
conn.send("git submodule update --init\n")
|
||||||
conn.send("git submodule foreach --recursive git reset --hard\n")
|
conn.send("git submodule foreach --recursive git reset --hard\n")
|
||||||
conn.send("git submodule foreach --recursive git clean -xdf\n")
|
conn.send("git submodule foreach --recursive git clean -xdf\n")
|
||||||
conn.send("echo \"git took $SECONDS seconds\"\n")
|
conn.send('echo "git took $SECONDS seconds"\n')
|
||||||
|
|
||||||
test_func(conn)
|
conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")
|
||||||
|
|
||||||
|
# run the test
|
||||||
|
conn.send(test_cmd + "\n")
|
||||||
|
|
||||||
|
# get the result and print it back out
|
||||||
conn.send('echo "RESULT:" $?\n')
|
conn.send('echo "RESULT:" $?\n')
|
||||||
conn.send("exit\n")
|
conn.send("exit\n")
|
||||||
return conn
|
|
||||||
|
|
||||||
def test_modeld(conn):
|
|
||||||
conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
eon_name = os.environ.get('eon_name', None)
|
|
||||||
|
|
||||||
conn = run_test(eon_name, test_modeld)
|
|
||||||
|
|
||||||
dat = b""
|
dat = b""
|
||||||
|
conn.settimeout(120)
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
recvd = conn.recv(4096)
|
try:
|
||||||
|
recvd = conn.recv(4096)
|
||||||
|
except socket.timeout:
|
||||||
|
print("connection to phone timed out")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
if len(recvd) == 0:
|
if len(recvd) == 0:
|
||||||
break
|
break
|
||||||
|
|
||||||
|
@ -71,5 +85,9 @@ if __name__ == "__main__":
|
||||||
sys.stdout.buffer.write(recvd)
|
sys.stdout.buffer.write(recvd)
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
|
||||||
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
|
return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
|
||||||
sys.exit(int(returns[0]))
|
sys.exit(return_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run_on_phone(sys.argv[1])
|
||||||
|
|
Loading…
Reference in New Issue