Fast CI (#1537)

* don't compile in docker build stage * disable that for now * do we need that? * upload artifact already compresses * that wasn't faster This reverts commit 2221e13c71. * try that * don't think that's used * this too * check this * less layers * that's not in the dockerfile anymore * system * even less layers and clear pipenv cache * no inline comment i guess * uninstall pipenv after we're done with it * separate build job adds too much overhead * quotes * clean that up * increase timeout * cleanup
2020-05-19 02:12:57 -07:00 · 2020-05-19 02:12:57 -07:00 · 45ee131a44
parent 8d46c0fa8a
commit 45ee131a44
2 changed files with 90 additions and 93 deletions
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@ -1,37 +1,21 @@
-name: Openpilot Tests
+name: openpilot tests
 on: [push, pull_request]

 env:
  RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
  PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
-  LOAD: docker load -i tmppilot.tar.gz/tmppilot.tar.gz
  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
  UNIT_TEST: coverage run --append -m unittest discover
+  BUILD: |
+      docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
+      docker pull docker.io/commaai/openpilot:latest || true
+      docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .

 jobs:
-  build:
-    name: build
-    runs-on: ubuntu-16.04
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-    - name: Build docker image
-      run: |
-        docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
-        docker pull docker.io/commaai/openpilot:latest || true
-
-        docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
-        docker save tmppilot:latest | gzip > tmppilot.tar.gz
-    - uses: actions/upload-artifact@v2
-      with:
-        name: tmppilot.tar.gz
-        path: tmppilot.tar.gz
-
  build_release:
    name: build release
    runs-on: ubuntu-16.04
-    timeout-minutes: 30
+    timeout-minutes: 50
    env:
      TEST_DIR: tmppilot
    steps:
@ -56,41 +40,37 @@ jobs:
           .coveragerc-app $TEST_DIR
        cd $TEST_DIR
        mkdir pyextra laika laika_repo tools release
-    - name: Build
-      run: |
-        cd $TEST_DIR
-        docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
-        docker pull docker.io/commaai/openpilot:latest || true
-        docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
+    - name: Build Docker image
+      run: cd $TEST_DIR && eval "$BUILD"
+    - name: Build openpilot
+      run: $RUN "cd /tmp/openpilot && scons -j$(nproc)"

-  push:
-    name: push
+  docker_push:
+    name: docker push
    runs-on: ubuntu-16.04
-    needs: build
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: linter # hack to ensure slow tests run first since this and linter are fast
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
-      - name: Login to dockerhub
-        run: docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Tag image
-        run: docker tag tmppilot docker.io/commaai/openpilot:latest
-      - name: Push image
-        run: docker push docker.io/commaai/openpilot:latest
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Push to dockerhub
+        run: |
+          docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker tag tmppilot docker.io/commaai/openpilot:latest
+          docker push docker.io/commaai/openpilot:latest

  linter:
    name: linter
    runs-on: ubuntu-16.04
-    needs: build
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
      - name: flake8
        run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
      - name: pylint
@ -99,16 +79,16 @@ jobs:
  unit_tests:
    name: unit tests
    runs-on: ubuntu-16.04
-    needs: build
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
      - name: Run unit tests
        run: |
          $PERSIST "cd /tmp/openpilot && \
+                    scons -j$(nproc) && \
                    coverage run selfdrive/test/test_fingerprints.py && \
                    $UNIT_TEST common && \
                    $UNIT_TEST opendbc/can && \
@ -127,17 +107,18 @@ jobs:
  process_replay:
    name: process replay
    runs-on: ubuntu-16.04
-    needs: build
-    timeout-minutes: 30
+    timeout-minutes: 50
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
      - name: Run replay
        run: |
-          $PERSIST "cd /tmp/openpilot && CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
+          $PERSIST "cd /tmp/openpilot && \
+                    scons -j$(nproc) && \
+                    CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
      - name: Upload coverage to Codecov
        run: |
          docker commit tmppilot tmppilotci
@ -158,17 +139,20 @@ jobs:
  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-16.04
-    needs: build
-    timeout-minutes: 30
+    timeout-minutes: 50
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
      - name: Test longitudinal
        run: |
-          $PERSIST "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
+          $PERSIST "mkdir -p /tmp/openpilot/selfdrive/test/out && \
+                    cd /tmp/openpilot/ && \
+                    scons -j$(nproc) && \
+                    cd selfdrive/test/longitudinal_maneuvers && \
+                    OPTEST=1 ./test_longitudinal.py"
      - name: Copy artifacts
        if: always()
        run: |
@ -183,17 +167,20 @@ jobs:
  test_car_models:
    name: test car models
    runs-on: ubuntu-16.04
-    needs: build
-    timeout-minutes: 30
+    timeout-minutes: 50
    steps:
-      - uses: actions/download-artifact@v1
+      - uses: actions/checkout@v2
        with:
-          name: tmppilot.tar.gz
-      - name: Load image
-        run: $LOAD
+          submodules: true
+      - name: Build Docker image
+        run: eval "$BUILD"
      - name: Test car models
        run: |
-          $PERSIST "mkdir -p /data/params && cd /tmp/openpilot && coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && coverage combine"
+          $PERSIST "mkdir -p /data/params && \
+                    cd /tmp/openpilot && \
+                    scons -j$(nproc) && \
+                    coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && \
+                    coverage combine"
      - name: Upload coverage to Codecov
        run: |
          docker commit tmppilot tmppilotci
--- a/Dockerfile.openpilot
+++ b/Dockerfile.openpilot
@ -1,5 +1,6 @@
 FROM ubuntu:16.04
 ENV PYTHONUNBUFFERED 1
+ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}

 RUN apt-get update && apt-get install -y --no-install-recommends \
    autoconf \
@ -22,12 +23,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    libglfw3-dev \
    libglib2.0-0 \
    liblzma-dev \
-    libmysqlclient-dev \
    libomp-dev \
    libopencv-dev \
    libssl-dev \
    libsqlite3-dev \
-    libtool \
    libusb-1.0-0-dev \
    libczmq-dev \
    libzmq3-dev \
@ -39,33 +38,49 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
    python-pip \
    sudo \
    wget \
-    && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/*

 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8

+# Install python dependencies
 RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-
 ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-RUN pyenv install 3.8.2 && pyenv global 3.8.2
-RUN pyenv rehash
-
-RUN pip install --no-cache-dir pipenv==2018.11.26

 COPY Pipfile Pipfile.lock /tmp/
-
-RUN cd /tmp && pipenv install --system --deploy
-
-# Install subset of dev dependencies needed for CI
-RUN pip install --no-cache-dir matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3 coverage==5.1
-
-ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
+RUN pyenv install 3.8.2 && \
+    pyenv global 3.8.2 && \
+    pyenv rehash && \
+    pip install --no-cache-dir pipenv==2018.11.26 && \
+    cd /tmp && \
+    pipenv install --system --deploy --clear && \
+    pip uninstall -y pipenv && \
+    pip install --no-cache-dir \
+      matplotlib==3.1.1 \
+      dictdiffer==0.8.0 \
+      fastcluster==1.1.25 \
+      aenum==2.2.1 \
+      lru-dict==1.1.6 \
+      scipy==1.4.1 \
+      tenacity==5.1.1 \
+      azure-common==1.1.23 \
+      azure-nspkg==3.0.2 \
+      azure-storage-blob==2.1.0 \
+      azure-storage-common==2.1.0 \
+      azure-storage-nspkg==3.1.0 \
+      pycurl==7.43.0.3 \
+      coverage==5.1

 RUN mkdir -p /tmp/openpilot

-COPY ./flake8_openpilot.sh ./pylint_openpilot.sh ./.pylintrc ./.coveragerc-app /tmp/openpilot/
+COPY SConstruct \
+     flake8_openpilot.sh \
+     pylint_openpilot.sh \
+     .pylintrc \
+     .coveragerc-app \
+     /tmp/openpilot/

 COPY ./pyextra /tmp/openpilot/pyextra
 COPY ./phonelibs /tmp/openpilot/phonelibs
@ -79,8 +94,3 @@ COPY ./opendbc /tmp/openpilot/opendbc
 COPY ./cereal /tmp/openpilot/cereal
 COPY ./panda /tmp/openpilot/panda
 COPY ./selfdrive /tmp/openpilot/selfdrive
-
-COPY SConstruct /tmp/openpilot/SConstruct
-
-RUN mkdir -p /tmp/openpilot/selfdrive/test/out
-RUN cd /tmp/openpilot && scons -j$(nproc)