spacecruft
/
nopenpilot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

On-device CI framework (#1784) 

* let's see if this works

* fix build_release actions job

* does jenkins like this config

* separate jenkinsfile for release build

* fix devel build

* devel build should work

* always pass that for now

* run modeld replay

* release2 build will be a separate PR

* pass env to phone shell

* force checkout

* run on real jenkins eons

* add timeout

* rsync

* more timeout

* trailing slash

* fix branch detection

* debug

* not sure why paramiko doesn't pass it through

* newline

* CI_PUSH

* still not passing it

* test branch

* should be good now
pull/1815/head
Adeeb Shihadeh 2020-07-03 19:12:05 -07:00 committed by GitHub
parent d417481e2b
commit 32f03ec8a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 76 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
Jenkinsfile vendored
View File

@ -4,38 +4,42 @@ pipeline {
image 'python:3.7.3'
args '--user=root'
}
}
environment {
COMMA_JWT = credentials('athena-test-jwt')
CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
}
stages {
stage('Device Tests') {
stage('On-device Tests') {
parallel {
stage('Build/Test') {
stage('Build') {
steps {
lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_name', quantity: 1){
lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
timeout(time: 30, unit: 'MINUTES') {
dir(path: 'release') {
sh 'pip install paramiko'
sh 'python remote_build.py'
}
}
}
}
}
stage('Replay Tests') {
steps {
lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_name', quantity: 1){
timeout(time: 45, unit: 'MINUTES') {
dir(path: 'selfdrive/test') {
sh 'pip install paramiko'
sh 'python phone_ci.py'
sh 'python phone_ci.py "cd release && ./build_devel.sh"'
}
}
}
}
}
stage('Replay Tests') {
steps {
lock(resource: "", label: 'eon2', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
timeout(time: 60, unit: 'MINUTES') {
dir(path: 'selfdrive/test') {
sh 'pip install paramiko'
sh 'python phone_ci.py "cd selfdrive/test/process_replay && ./camera_replay.py"'
}
}
}
}
}
}
}
}

5
SConstruct
View File

@ -161,7 +161,10 @@ env = Environment(
)
if os.environ.get('SCONS_CACHE'):
CacheDir('/tmp/scons_cache')
if QCOM_REPLAY:
CacheDir('/tmp/scons_cache_qcom_replay')
else:
CacheDir('/tmp/scons_cache')
node_interval = 5
node_count = 0

8
release/build_devel.sh
View File

@ -37,7 +37,7 @@ echo "[-] bringing master-ci and devel in sync T=$SECONDS"
git fetch origin master-ci
git fetch origin devel
git checkout --track origin/master-ci || true
git checkout -f --track origin/master-ci
git reset --hard master-ci
git checkout master-ci
git reset --hard origin/devel
@ -93,9 +93,9 @@ pushd panda/board/pedal
make obj/comma.bin
popd
if [ ! -z "$PUSH" ]; then
echo "[-] Pushing to $PUSH T=$SECONDS"
git push -f origin master-ci:$PUSH
if [ ! -z "$CI_PUSH" ]; then
echo "[-] Pushing to $CI_PUSH T=$SECONDS"
git push -f origin master-ci:$CI_PUSH
fi
echo "[-] done pushing T=$SECONDS"

12
release/files_common
View File

@ -1,16 +1,16 @@
README.md
SAFETY.md
.gitignore
LICENSE
launch_chffrplus.sh
launch_openpilot.sh
CONTRIBUTING.md
RELEASES.md
Jenkinsfile
SConstruct
CONTRIBUTING.md
README.md
RELEASES.md
SAFETY.md
apk/ai.comma*.apk
common/.gitignore

73
release/remote_build.py
View File

@ -1,73 +0,0 @@
#!/usr/bin/env python2
import paramiko # pylint: disable=import-error
import os
import sys
import re
import time
import socket
def start_build(name):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key_file = open(os.path.join(os.path.dirname(__file__), "../tools/ssh/key/id_rsa"))
key = paramiko.RSAKey.from_private_key(key_file)
print("SSH to phone {}".format(name))
# Try connecting for one minute
t_start = time.time()
while True:
try:
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
print("Connection failed")
if time.time() - t_start > 60:
raise
else:
break
time.sleep(1)
conn = ssh.invoke_shell()
branch = os.environ['GIT_BRANCH']
commit = os.environ.get('GIT_COMMIT', branch)
conn.send('uname -a\n')
conn.send('cd /data/openpilot_source\n')
conn.send("git reset --hard\n")
conn.send("git fetch origin\n")
conn.send("git checkout %s\n" % commit)
conn.send("git clean -xdf\n")
conn.send("git submodule update --init\n")
conn.send("git submodule foreach --recursive git reset --hard\n")
conn.send("git submodule foreach --recursive git clean -xdf\n")
conn.send("echo \"git took $SECONDS seconds\"\n")
push = "PUSH=master-ci" if branch == "master" else ""
conn.send("%s /data/openpilot_source/release/build_devel.sh\n" % push)
conn.send('echo "RESULT:" $?\n')
conn.send("exit\n")
return conn
if __name__ == "__main__":
eon_name = os.environ.get('eon_name', None)
conn = start_build(eon_name)
dat = b""
while True:
recvd = conn.recv(4096)
if len(recvd) == 0:
break
dat += recvd
sys.stdout.buffer.write(recvd)
sys.stdout.flush()
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
sys.exit(int(returns[0]))

64
selfdrive/test/phone_ci.py
View File

@ -6,22 +6,29 @@ import re
import time
import socket
TEST_DIR = "/data/openpilotci"
def run_test(name, test_func):
SOURCE_DIR = "/data/openpilot_source/"
TEST_DIR = "/data/openpilot/"
def run_on_phone(test_cmd):
eon_ip = os.environ.get('eon_ip', None)
if eon_ip is None:
raise Exception("'eon_ip' not set")
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
key = paramiko.RSAKey.from_private_key(key_file)
print("SSH to phone {}".format(name))
print("SSH to phone at {}".format(eon_ip))
# Try connecting for one minute
# try connecting for one minute
t_start = time.time()
while True:
try:
ssh.connect(hostname=name, port=8022, pkey=key, timeout=10)
ssh.connect(hostname=eon_ip, port=8022, pkey=key, timeout=10)
except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError):
print("Connection failed")
if time.time() - t_start > 60:
@ -30,40 +37,47 @@ def run_test(name, test_func):
break
time.sleep(1)
conn = ssh.invoke_shell()
branch = os.environ['GIT_BRANCH']
commit = os.environ.get('GIT_COMMIT', branch)
conn.send("uname -a\n")
conn = ssh.invoke_shell()
conn.send(f"cd {TEST_DIR}\n")
# pass in all environment variables prefixed with 'CI_'
for k, v in os.environ.items():
if k.startswith("CI_"):
conn.send(f"export {k}='{v}'\n")
conn.send("export CI=1\n")
# set up environment
conn.send(f"cd {SOURCE_DIR}\n")
conn.send("git reset --hard\n")
conn.send("git fetch origin\n")
conn.send("git checkout %s\n" % commit)
conn.send(f"git checkout {commit}\n")
conn.send("git clean -xdf\n")
conn.send("git submodule update --init\n")
conn.send("git submodule foreach --recursive git reset --hard\n")
conn.send("git submodule foreach --recursive git clean -xdf\n")
conn.send("echo \"git took $SECONDS seconds\"\n")
conn.send('echo "git took $SECONDS seconds"\n')
test_func(conn)
conn.send(f"rsync -a --delete {SOURCE_DIR} {TEST_DIR}\n")
# run the test
conn.send(test_cmd + "\n")
# get the result and print it back out
conn.send('echo "RESULT:" $?\n')
conn.send("exit\n")
return conn
def test_modeld(conn):
conn.send(f"cd selfdrive/test/process_replay && PYTHONPATH={TEST_DIR} ./camera_replay.py\n")
if __name__ == "__main__":
eon_name = os.environ.get('eon_name', None)
conn = run_test(eon_name, test_modeld)
dat = b""
conn.settimeout(120)
while True:
recvd = conn.recv(4096)
try:
recvd = conn.recv(4096)
except socket.timeout:
print("connection to phone timed out")
sys.exit(1)
if len(recvd) == 0:
break
@ -71,5 +85,9 @@ if __name__ == "__main__":
sys.stdout.buffer.write(recvd)
sys.stdout.flush()
returns = re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)
sys.exit(int(returns[0]))
return_code = int(re.findall(rb'^RESULT: (\d+)', dat[-1024:], flags=re.MULTILINE)[0])
sys.exit(return_code)
if __name__ == "__main__":
run_on_phone(sys.argv[1])