Build release2 and dashcam in CI (#1834)

* start building release2 in CI

* build script

* bring over changes from test branch

* fix build release test

* remove references to test branches

* ucnomment push

* fix duplicate remove

* two commits for dashcam

* need the key

.github/workflows/test.yaml

@@ -39,7 +39,7 @@ jobs:
         # need these so docker copy won't fail
         cp Pipfile Pipfile.lock .pylintrc .coveragerc-app .pre-commit-config.yaml $TEST_DIR
         cd $TEST_DIR
-        mkdir laika laika_repo tools release
+        mkdir laika laika_repo tools
     - name: Build Docker image
       run: cd $TEST_DIR && eval "$BUILD"
     - name: Build openpilot and run quick check

Jenkinsfile

@@ -7,14 +7,42 @@ pipeline {
   }
   environment {
     COMMA_JWT = credentials('athena-test-jwt')
-    CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
   }
 
   stages {
+
+    stage('Release Build') {
+      when {
+        branch 'devel-staging'
+      }
+      steps {
+        lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
+          timeout(time: 60, unit: 'MINUTES') {
+            dir(path: 'selfdrive/test') {
+              sh 'pip install paramiko'
+              sh 'python phone_ci.py "cd release && PUSH=1 ./build_release2.sh"'
+            }
+          }
+        }
+      }
+    }
+
     stage('On-device Tests') {
+      when {
+        not {
+          anyOf {
+            branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging'
+          }
+        }
+      }
+
       parallel {
 
         stage('Build') {
+          environment {
+            CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ''}"
+          }
+
           steps {
             lock(resource: "", label: 'eon', inversePrecedence: true, variable: 'eon_ip', quantity: 1){
               timeout(time: 30, unit: 'MINUTES') {

release/build_release2.sh

@@ -8,24 +8,32 @@ export GIT_AUTHOR_EMAIL="user@comma.ai"
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
-# Create folders
-rm -rf /data/openpilot
-mkdir -p /data/openpilot
-cd /data/openpilot
+# set CLEAN to build outside of CI
+if [ ! -z "$CLEAN" ]; then
+  # Create folders
+  rm -rf /data/openpilot
+  mkdir -p /data/openpilot
+  cd /data/openpilot
+
+  # Create git repo
+  git init
+  git remote add origin git@github.com:commaai/openpilot.git
+  git fetch origin devel-staging
+else
+  cd /data/openpilot
+  git clean -xdf
+  git branch -D release2-staging || true
+fi
 
-# Create git repo
-git init
-git remote add origin git@github.com:commaai/openpilot.git
-git fetch origin devel
 git fetch origin release2-staging
 git fetch origin dashcam-staging
 
-# Checkout devel
-#git checkout origin/devel
-#git clean -xdf
-
 # Create release2 with no history
-git checkout --orphan release2-staging origin/devel
+if [ ! -z "$CLEAN" ]; then
+  git checkout --orphan release2-staging origin/devel-staging
+else
+  git checkout --orphan release2-staging
+fi
 
 VERSION=$(cat selfdrive/common/version.h | awk -F\" '{print $2}')
 git commit -m "openpilot v$VERSION"
@@ -55,7 +63,7 @@ find . -name '*.o' -delete
 find . -name '*.os' -delete
 find . -name '*.pyc' -delete
 find . -name '__pycache__' -delete
-rm .sconsign.dblite
+rm -rf .sconsign.dblite Jenkinsfile release/
 
 # Restore phonelibs
 git checkout phonelibs/
@@ -70,11 +78,15 @@ git commit --amend -m "openpilot v$VERSION"
 # Print committed files that are normally gitignored
 #git status --ignored
 
-# Push to release2-staging
-git push -f origin release2-staging
+if [ ! -z "$PUSH" ]; then
+  git remote set-url origin git@github.com:commaai/openpilot.git
 
-# Create dashcam release
-git rm selfdrive/car/*/carcontroller.py
+  # Push to release2-staging
+  git push -f origin release2-staging
 
-git commit -m "create dashcam release from release2"
-git push -f origin release2-staging:dashcam-staging
+  # Create dashcam release
+  git rm selfdrive/car/*/carcontroller.py
+
+  git commit -m "create dashcam release from release2"
+  git push -f origin release2-staging:dashcam-staging
+fi

release/files_common

@@ -58,6 +58,8 @@ common/api/__init__.py
 models/supercombo.dlc
 models/dmonitoring_model_q.dlc
 
+release/build_release2.sh
+
 selfdrive/version.py
 
 selfdrive/__init__.py
@@ -314,10 +316,11 @@ selfdrive/thermald/thermald.py
 selfdrive/thermald/power_monitoring.py
 
 selfdrive/test/__init__.py
+selfdrive/test/id_rsa
 selfdrive/test/helpers.py
+selfdrive/test/phone_ci.py
 selfdrive/test/test_openpilot.py
 selfdrive/test/test_fingerprints.py
-selfdrive/test/test_car_models.py
 selfdrive/test/test_cpu_usage.py
 
 selfdrive/ui/SConscript

selfdrive/test/id_rsa

@@ -0,0 +1,28 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
+Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
+t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
+Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
+acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
+1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
+rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
+x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
+gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
+Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
+LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
+s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
+Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
+6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
+uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
+0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
+FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
+ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
+4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
+nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
+VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
+s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
+jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
+b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
+RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
+6co17RFaAZHwGfCFFjO76Q==
+-----END RSA PRIVATE KEY-----

selfdrive/test/phone_ci.py

@@ -19,7 +19,7 @@ def run_on_phone(test_cmd):
   ssh = paramiko.SSHClient()
   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-  key_file = open(os.path.join(os.path.dirname(__file__), "../../tools/ssh/key/id_rsa"))
+  key_file = open(os.path.join(os.path.dirname(__file__), "id_rsa"))
   key = paramiko.RSAKey.from_private_key(key_file)
 
   print("SSH to phone at {}".format(eon_ip))
@@ -52,6 +52,8 @@ def run_on_phone(test_cmd):
   conn.send(f"cd {SOURCE_DIR}\n")
   conn.send("git reset --hard\n")
   conn.send("git fetch origin\n")
+  conn.send("find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \\;\n")
+  conn.send(f"git reset --hard {commit}\n")
   conn.send(f"git checkout {commit}\n")
   conn.send("git clean -xdf\n")
   conn.send("git submodule update --init\n")

tools/ssh/key/id_rsa

@@ -1,28 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC+iXXq30Tq+J5N
-Kat3KWHCzcmwZ55nGh6WggAqECa5CasBlM9VeROpVu3beA+5h0MibRgbD4DMtVXB
-t6gEvZ8nd04E7eLA9LTZyFDZ7SkSOVj4oXOQsT0GnJmKrASW5KslTWqVzTfo2XCt
-Z+004ikLxmyFeBO8NOcErW1pa8gFdQDToH9FrA7kgysic/XVESTOoe7XlzRoe/eZ
-acEQ+jtnmFd21A4aEADkk00Ahjr0uKaJiLUAPatxs2icIXWpgYtfqqtaKF23wSt6
-1OTu6cAwXbOWr3m+IUSRUO0IRzEIQS3z1jfd1svgzSgSSwZ1Lhj4AoKxIEAIc8qJ
-rO4uymCJAgMBAAECggEBAISFevxHGdoL3Z5xkw6oO5SQKO2GxEeVhRzNgmu/HA+q
-x8OryqD6O1CWY4037kft6iWxlwiLOdwna2P25ueVM3LxqdQH2KS4DmlCx+kq6FwC
-gv063fQPMhC9LpWimvaQSPEC7VUPjQlo4tPY6sTTYBUOh0A1ihRm/x7juKuQCWix
-Cq8C/DVnB1X4mGj+W3nJc5TwVJtgJbbiBrq6PWrhvB/3qmkxHRL7dU2SBb2iNRF1
-LLY30dJx/cD73UDKNHrlrsjk3UJc29Mp4/MladKvUkRqNwlYxSuAtJV0nZ3+iFkL
-s3adSTHdJpClQer45R51rFDlVsDz2ZBpb/hRNRoGDuECgYEA6A1EixLq7QYOh3cb
-Xhyh3W4kpVvA/FPfKH1OMy3ONOD/Y9Oa+M/wthW1wSoRL2n+uuIW5OAhTIvIEivj
-6bAZsTT3twrvOrvYu9rx9aln4p8BhyvdjeW4kS7T8FP5ol6LoOt2sTP3T1LOuJPO
-uQvOjlKPKIMh3c3RFNWTnGzMPa0CgYEA0jNiPLxP3A2nrX0keKDI+VHuvOY88gdh
-0W5BuLMLovOIDk9aQFIbBbMuW1OTjHKv9NK+Lrw+YbCFqOGf1dU/UN5gSyE8lX/Q
-FsUGUqUZx574nJZnOIcy3ONOnQLcvHAQToLFAGUd7PWgP3CtHkt9hEv2koUwL4vo
-ikTP1u9Gkc0CgYEA2apoWxPZrY963XLKBxNQecYxNbLFaWq67t3rFnKm9E8BAICi
-4zUaE5J1tMVi7Vi9iks9Ml9SnNyZRQJKfQ+kaebHXbkyAaPmfv+26rqHKboA0uxA
-nDOZVwXX45zBkp6g1sdHxJx8JLoGEnkC9eyvSi0C//tRLx86OhLErXwYcNkCf1it
-VMRKrWYoXJTUNo6tRhvodM88UnnIo3u3CALjhgU4uC1RTMHV4ZCGBwiAOb8GozSl
-s5YD1E1iKwEULloHnK6BIh6P5v8q7J6uf/xdqoKMjlWBHgq6/roxKvkSPA1DOZ3l
-jTadcgKFnRUmc+JT9p/ZbCxkA/ALFg8++G+0ghECgYA8vG3M/utweLvq4RI7l7U7
-b+i2BajfK2OmzNi/xugfeLjY6k2tfQGRuv6ppTjehtji2uvgDWkgjJUgPfZpir3I
-RsVMUiFgloWGHETOy0Qvc5AwtqTJFLTD1Wza2uBilSVIEsg6Y83Gickh+ejOmEsY
-6co17RFaAZHwGfCFFjO76Q==
------END RSA PRIVATE KEY-----

tools/ssh/key/id_rsa

@@ -0,0 +1 @@
+../../../selfdrive/test/id_rsa