Improve CPU usage test reliability (#2002)
* run phone tests in parallel * better cpu test * re-enable test * no root * terms version * not one * yes * debug * that's coverd by min cpu percentpull/2000/head
parent
55ece6991e
commit
ecf0a8c8d4
|
@ -36,16 +36,31 @@ pipeline {
|
||||||
}
|
}
|
||||||
|
|
||||||
stages {
|
stages {
|
||||||
|
|
||||||
|
stage('Release Build') {
|
||||||
|
when {
|
||||||
|
branch 'devel-staging'
|
||||||
|
}
|
||||||
|
steps {
|
||||||
|
phone_steps("eon-build", [
|
||||||
|
["build release2-staging and dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"],
|
||||||
|
])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
stage('openpilot tests') {
|
stage('openpilot tests') {
|
||||||
when {
|
when {
|
||||||
not {
|
not {
|
||||||
anyOf {
|
anyOf {
|
||||||
branch 'master-ci'; branch 'devel'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'
|
branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
parallel {
|
|
||||||
|
|
||||||
|
|
||||||
|
stages {
|
||||||
|
|
||||||
|
/*
|
||||||
stage('PC tests') {
|
stage('PC tests') {
|
||||||
agent {
|
agent {
|
||||||
dockerfile {
|
dockerfile {
|
||||||
|
@ -67,6 +82,7 @@ pipeline {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
*/
|
||||||
|
|
||||||
stage('On-device Tests') {
|
stage('On-device Tests') {
|
||||||
agent {
|
agent {
|
||||||
|
@ -75,18 +91,10 @@ pipeline {
|
||||||
args '--user=root'
|
args '--user=root'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
stages {
|
|
||||||
|
|
||||||
stage('Release Build') {
|
stages {
|
||||||
when {
|
stage('parallel tests') {
|
||||||
branch 'devel-staging'
|
parallel {
|
||||||
}
|
|
||||||
steps {
|
|
||||||
phone_steps("eon-build", [
|
|
||||||
["build release2-staging and dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"],
|
|
||||||
])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stage('Devel Build') {
|
stage('Devel Build') {
|
||||||
environment {
|
environment {
|
||||||
|
@ -96,7 +104,7 @@ pipeline {
|
||||||
phone_steps("eon", [
|
phone_steps("eon", [
|
||||||
["build devel", "cd release && CI_PUSH=${env.CI_PUSH} ./build_devel.sh"],
|
["build devel", "cd release && CI_PUSH=${env.CI_PUSH} ./build_devel.sh"],
|
||||||
["test openpilot", "nosetests -s selfdrive/test/test_openpilot.py"],
|
["test openpilot", "nosetests -s selfdrive/test/test_openpilot.py"],
|
||||||
//["test cpu usage", "cd selfdrive/test/ && ./test_cpu_usage.py"],
|
["test cpu usage", "cd selfdrive/test/ && ./test_cpu_usage.py"],
|
||||||
["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
|
["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
|
||||||
])
|
])
|
||||||
}
|
}
|
||||||
|
@ -121,7 +129,8 @@ pipeline {
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,12 +5,12 @@ from nose.tools import nottest
|
||||||
from common.android import ANDROID
|
from common.android import ANDROID
|
||||||
from common.apk import update_apks, start_offroad, pm_apply_packages, android_packages
|
from common.apk import update_apks, start_offroad, pm_apply_packages, android_packages
|
||||||
from common.params import Params
|
from common.params import Params
|
||||||
from selfdrive.version import training_version
|
from selfdrive.version import training_version, terms_version
|
||||||
from selfdrive.manager import start_managed_process, kill_managed_process, get_running
|
from selfdrive.manager import start_managed_process, kill_managed_process, get_running
|
||||||
|
|
||||||
def set_params_enabled():
|
def set_params_enabled():
|
||||||
params = Params()
|
params = Params()
|
||||||
params.put("HasAcceptedTerms", "1")
|
params.put("HasAcceptedTerms", terms_version)
|
||||||
params.put("HasCompletedSetup", "1")
|
params.put("HasCompletedSetup", "1")
|
||||||
params.put("OpenpilotEnabledToggle", "1")
|
params.put("OpenpilotEnabledToggle", "1")
|
||||||
params.put("CommunityFeaturesToggle", "1")
|
params.put("CommunityFeaturesToggle", "1")
|
||||||
|
|
|
@ -1,13 +1,12 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
import threading
|
|
||||||
import _thread
|
|
||||||
import signal
|
|
||||||
import sys
|
import sys
|
||||||
|
import subprocess
|
||||||
|
|
||||||
import cereal.messaging as messaging
|
import cereal.messaging as messaging
|
||||||
|
from common.basedir import BASEDIR
|
||||||
from common.params import Params
|
from common.params import Params
|
||||||
import selfdrive.manager as manager
|
|
||||||
from selfdrive.test.helpers import set_params_enabled
|
from selfdrive.test.helpers import set_params_enabled
|
||||||
|
|
||||||
def cputime_total(ct):
|
def cputime_total(ct):
|
||||||
|
@ -40,7 +39,7 @@ def print_cpu_usage(first_proc, last_proc):
|
||||||
("./logcatd", 0),
|
("./logcatd", 0),
|
||||||
]
|
]
|
||||||
|
|
||||||
r = 0
|
r = True
|
||||||
dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9
|
dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9
|
||||||
result = "------------------------------------------------\n"
|
result = "------------------------------------------------\n"
|
||||||
for proc_name, normal_cpu_usage in procs:
|
for proc_name, normal_cpu_usage in procs:
|
||||||
|
@ -51,26 +50,25 @@ def print_cpu_usage(first_proc, last_proc):
|
||||||
cpu_usage = cpu_time / dt * 100.
|
cpu_usage = cpu_time / dt * 100.
|
||||||
if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
|
if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0):
|
||||||
result += f"Warning {proc_name} using more CPU than normal\n"
|
result += f"Warning {proc_name} using more CPU than normal\n"
|
||||||
r = 1
|
r = False
|
||||||
elif cpu_usage < min(normal_cpu_usage * 0.3, max(normal_cpu_usage - 1.0, 0.0)):
|
elif cpu_usage < min(normal_cpu_usage * 0.3, max(normal_cpu_usage - 1.0, 0.0)):
|
||||||
result += f"Warning {proc_name} using less CPU than normal\n"
|
result += f"Warning {proc_name} using less CPU than normal\n"
|
||||||
r = 1
|
r = False
|
||||||
result += f"{proc_name.ljust(35)} {cpu_usage:.2f}%\n"
|
result += f"{proc_name.ljust(35)} {cpu_usage:.2f}%\n"
|
||||||
except IndexError:
|
except IndexError:
|
||||||
result += f"{proc_name.ljust(35)} NO METRICS FOUND\n"
|
result += f"{proc_name.ljust(35)} NO METRICS FOUND\n"
|
||||||
r = 1
|
r = False
|
||||||
result += "------------------------------------------------\n"
|
result += "------------------------------------------------\n"
|
||||||
print(result)
|
print(result)
|
||||||
return r
|
return r
|
||||||
|
|
||||||
def all_running():
|
def test_cpu_usage():
|
||||||
running = manager.get_running()
|
cpu_ok = False
|
||||||
return all(p in running and running[p].is_alive() for p in manager.car_started_processes)
|
|
||||||
|
|
||||||
return_code = 1
|
# start manager
|
||||||
def test_thread():
|
manager_path = os.path.join(BASEDIR, "selfdrive/manager.py")
|
||||||
|
manager_proc = subprocess.Popen(["python", manager_path])
|
||||||
try:
|
try:
|
||||||
global return_code
|
|
||||||
proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000)
|
proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000)
|
||||||
|
|
||||||
# wait until everything's started and get first sample
|
# wait until everything's started and get first sample
|
||||||
|
@ -80,33 +78,26 @@ def test_thread():
|
||||||
break
|
break
|
||||||
time.sleep(2)
|
time.sleep(2)
|
||||||
first_proc = messaging.recv_sock(proc_sock, wait=True)
|
first_proc = messaging.recv_sock(proc_sock, wait=True)
|
||||||
if first_proc is None or not all_running():
|
if first_proc is None:
|
||||||
err_msg = "procLog recv timed out" if first_proc is None else "all car started process not running"
|
raise Exception("\n\nTEST FAILED: progLog recv timed out\n\n")
|
||||||
print(f"\n\nTEST FAILED: {err_msg}\n\n")
|
|
||||||
raise Exception
|
|
||||||
|
|
||||||
# run for a minute and get last sample
|
# run for a minute and get last sample
|
||||||
time.sleep(60)
|
time.sleep(60)
|
||||||
last_proc = messaging.recv_sock(proc_sock, wait=True)
|
last_proc = messaging.recv_sock(proc_sock, wait=True)
|
||||||
return_code = print_cpu_usage(first_proc, last_proc)
|
cpu_ok = print_cpu_usage(first_proc, last_proc)
|
||||||
if not all_running():
|
|
||||||
return_code = 1
|
|
||||||
finally:
|
finally:
|
||||||
_thread.interrupt_main()
|
manager_proc.terminate()
|
||||||
|
ret = manager_proc.wait(20)
|
||||||
|
if ret is None:
|
||||||
|
manager_proc.kill()
|
||||||
|
return cpu_ok
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
|
|
||||||
# setup signal handler to exit with test status
|
|
||||||
def handle_exit(sig, frame):
|
|
||||||
sys.exit(return_code)
|
|
||||||
signal.signal(signal.SIGINT, handle_exit)
|
|
||||||
|
|
||||||
# start manager and test thread
|
|
||||||
set_params_enabled()
|
set_params_enabled()
|
||||||
Params().delete("CarParams")
|
Params().delete("CarParams")
|
||||||
|
|
||||||
t = threading.Thread(target=test_thread)
|
passed = False
|
||||||
t.daemon = True
|
try:
|
||||||
t.start()
|
passed = test_cpu_usage()
|
||||||
manager.main()
|
finally:
|
||||||
|
sys.exit(int(not passed))
|
||||||
|
|
Loading…
Reference in New Issue