openpilot/.github/workflows/test.yaml

name: tests
on:
  push:
    branches-ignore:
      - 'testing-closet*'
  pull_request:

env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai

  DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
  BUILD: |
      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .      

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
  UNIT_TEST: coverage run --append -m unittest discover

jobs:
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
      STRIPPED_DIR: tmppilot
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Strip non-release files
      run: |
        mkdir $STRIPPED_DIR
        cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
        cp Dockerfile.openpilot_base $STRIPPED_DIR

        # need this to build on x86
        cp -pR --parents phonelibs/libyuv phonelibs/snpe selfdrive/modeld/runners $STRIPPED_DIR        
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "cd /tmp/openpilot && \
                          scons -j$(nproc) && \
                          python -m unittest discover selfdrive/car"          

  build_mac:
    name: build macos
    runs-on: macos-10.15
    timeout-minutes: 60
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Determine pre-existing Homebrew packages
      if: steps.dependency-cache.outputs.cache-hit != 'true'
      run: |
        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
        ls -1 /usr/local/Cellar >> $GITHUB_ENV
        echo 'EOF' >> $GITHUB_ENV        
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: |
          ~/.pyenv
          ~/Library/Caches/pip
          ~/Library/Caches/pipenv
          /usr/local/Cellar
          ~/github_brew_cache_entries.txt          
        key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
    - name: Brew link restored dependencies
      if: steps.dependency-cache.outputs.cache-hit == 'true'
      run: |
        while read pkg; do
          brew link --force "$pkg" # `--force` for keg-only packages
        done < ~/github_brew_cache_entries.txt        
    - name: Install dependencies
      run: ./tools/mac_setup.sh
    - name: Build openpilot
      run: eval "$(pyenv init -)" && scons -j$(nproc)
    - name: Remove pre-existing Homebrew packages for caching
      if: steps.dependency-cache.outputs.cache-hit != 'true'
      run: |
        cd /usr/local/Cellar
        new_cellar=$(ls -1)
        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
            rm -rf "$pkg"
          fi
        done
        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt        

  build_webcam:
    name: build webcam
    runs-on: ubuntu-20.04
    timeout-minutes: 90
    env:
      IMAGE_NAME: openpilotwebcamci
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: |
        eval "$BUILD"
        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
        docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .        
    - name: Build openpilot
      run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
    - name: Push to container registry
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest        

  docker_push:
    name: docker push
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest        

  static_analysis:
    name: static analysis
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
      run: ${{ env.RUN }} "cd /tmp/openpilot/ && git init && git add -A && pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run valgrind
      run: |
        ${{ env.RUN }} "cd /tmp/openpilot && \
                        scons -j$(nproc) && \
                        FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"        
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt

  unit_tests:
    name: unit tests
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run unit tests
      run: |
        ${{ env.RUN }} "cd /tmp/openpilot && \
                        scons -j$(nproc) && \
                        coverage run selfdrive/test/test_fingerprints.py && \
                        $UNIT_TEST common && \
                        $UNIT_TEST opendbc/can && \
                        $UNIT_TEST selfdrive/boardd && \
                        $UNIT_TEST selfdrive/controls && \
                        $UNIT_TEST selfdrive/monitoring && \
                        $UNIT_TEST selfdrive/loggerd && \
                        $UNIT_TEST selfdrive/car && \
                        $UNIT_TEST selfdrive/locationd && \
                        $UNIT_TEST selfdrive/athena && \
                        $UNIT_TEST selfdrive/thermald && \
                        $UNIT_TEST tools/lib/tests"        
    - name: Upload coverage to Codecov
      run: bash <(curl -s https://codecov.io/bash) -v -F unit_tests

  process_replay:
    name: process replay
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run replay
      run: |
        ${{ env.RUN }} "cd /tmp/openpilot && \
                        scons -j$(nproc) && \
                        FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py"        
    - name: Upload coverage to Codecov
      run: bash <(curl -s https://codecov.io/bash) -v -F process_replay
    - name: Print diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt

  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test longitudinal
      run: |
        ${{ env.RUN }} "mkdir -p /tmp/openpilot/selfdrive/test/out && \
                        cd /tmp/openpilot/ && \
                        scons -j$(nproc) && \
                        cd selfdrive/test/longitudinal_maneuvers && \
                        ./test_longitudinal.py"        
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: longitudinal
        path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/

  test_car_models:
    name: car models
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/test.yaml', 'selfdrive/test/test_car_models.py') }}
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test car models
      run: |
        ${{ env.RUN }} "cd /tmp/openpilot && \
                        scons -j$(nproc) && \
                        FILEREADER_CACHE=1 coverage run --parallel-mode -m nose --processes=4 --process-timeout=60 \
                                           selfdrive/test/test_models.py && \
                        coverage combine"        
    - name: Upload coverage to Codecov
      run: bash <(curl -s https://codecov.io/bash) -v -F test_car_models