Integrate codecov.io (#1465)

* integrate codecov * add codecov to readme * don't need the token * fix badges * /bin/bash, not /bin/sh * needs coverage in the docker * from the git root * add CI_ENV like https://docs.codecov.io/docs/testing-with-docker * custom upload step * use codecov action * coverage xml * add unit tests, good luck * name the container tmppilot * add name, fix unit tests * yml, fix coveragerc-app * remove tmp/openpilot * try without xml * just call the upload script in docker * ugh, i hate docker * idk env * yaml? * don't be clever * maybe this * running out of ideas * coverage xml maybe * upload coverage.xml * ugh CI_RUN * unneeded artifact Co-authored-by: George Hotz <geohot@gmail.com>
2020-05-04 14:03:16 -07:00 · 2020-05-04 14:03:16 -07:00 · 4252028a84
parent d5ca841b0f
commit 4252028a84
8 changed files with 42 additions and 26 deletions
--- a/.coveragerc-app
+++ b/.coveragerc-app
@ -0,0 +1,3 @@
+[run]
+concurrency=multiprocessing
+
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@ -3,8 +3,10 @@ on: [push, pull_request]

 env:
  RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
+  PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
  LOAD: docker load -i tmppilot.tar.gz/tmppilot.tar.gz
-  UNIT_TEST: cd /tmp/openpilot && python -m unittest discover
+  CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
+  UNIT_TEST: coverage run --append -m unittest discover

 jobs:
  build:
@ -72,16 +74,21 @@ jobs:
        run: $LOAD
      - name: Run unit tests
        run: |
-          $RUN /tmp/openpilot/selfdrive/test/test_fingerprints.py
-          $RUN "$UNIT_TEST common"
-          $RUN "$UNIT_TEST opendbc/can"
-          $RUN "$UNIT_TEST selfdrive/boardd"
-          $RUN "$UNIT_TEST selfdrive/controls"
-          $RUN "$UNIT_TEST selfdrive/loggerd"
-          $RUN "$UNIT_TEST selfdrive/car"
-          $RUN "$UNIT_TEST selfdrive/locationd"
-          $RUN "$UNIT_TEST selfdrive/athena"
-          $RUN "$UNIT_TEST tools/lib/tests"
+          $PERSIST "cd /tmp/openpilot && \
+                    coverage run selfdrive/test/test_fingerprints.py && \
+                    $UNIT_TEST common && \
+                    $UNIT_TEST opendbc/can && \
+                    $UNIT_TEST selfdrive/boardd && \
+                    $UNIT_TEST selfdrive/controls && \
+                    $UNIT_TEST selfdrive/loggerd && \
+                    $UNIT_TEST selfdrive/car && \
+                    $UNIT_TEST selfdrive/locationd && \
+                    $UNIT_TEST selfdrive/athena && \
+                    $UNIT_TEST tools/lib/tests"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F unit_tests"

  process_replay:
    name: process replay
@ -96,14 +103,15 @@ jobs:
        run: $LOAD
      - name: Run replay
        run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
-          docker run --shm-size 1G --name ${CONTAINER_NAME} tmppilot /bin/sh -c "cd /tmp/openpilot/selfdrive/test/process_replay && CI=1 ./test_processes.py"
+          $PERSIST "cd /tmp/openpilot && CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F process_replay"
      - name: Copy diff
        if: always()
        run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
-          docker cp $CONTAINER_NAME:/tmp/openpilot/selfdrive/test/process_replay/diff.txt diff.txt
-          docker rm $CONTAINER_NAME
+          docker cp tmppilot:/tmp/openpilot/selfdrive/test/process_replay/diff.txt diff.txt
      - name: Print diff
        if: always()
        run: cat diff.txt
@ -126,15 +134,12 @@ jobs:
        run: $LOAD
      - name: Test longitudinal
        run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
-          docker run --shm-size 1G --name ${CONTAINER_NAME} tmppilot /bin/sh -c "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
+          $PERSIST "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
      - name: Copy artifacts
        if: always()
        run: |
-          CONTAINER_NAME="tmppilot_${GITHUB_SHA}"
          mkdir out
-          docker cp $CONTAINER_NAME:/tmp/openpilot/selfdrive/test/longitudinal_maneuvers/out/longitudinal/ out/
-          docker rm $CONTAINER_NAME
+          docker cp tmppilot:/tmp/openpilot/selfdrive/test/longitudinal_maneuvers/out/longitudinal/ out/
      - uses: actions/upload-artifact@v1
        if: always()
        with:
@ -153,4 +158,10 @@ jobs:
      - name: Load image
        run: $LOAD
      - name: Test car models
-        run: $RUN "mkdir -p /data/params && cd /tmp/openpilot/selfdrive/test/ && ./test_car_models.py"
+        run: |
+          $PERSIST "mkdir -p /data/params && cd /tmp/openpilot && coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && coverage combine"
+      - name: Upload coverage to Codecov
+        run: |
+          docker commit tmppilot tmppilotci
+          $CI_RUN "cd /tmp/openpilot && bash <(curl -s https://codecov.io/bash) -Z -F test_car_models"
+
--- a/.gitignore
+++ b/.gitignore
@ -56,6 +56,7 @@ xx
 panda_jungle

 .coverage*
+coverage.xml
 htmlcov
 pandaextra

--- a/Dockerfile.openpilot
+++ b/Dockerfile.openpilot
@ -61,7 +61,7 @@ RUN python --version
 RUN cd /tmp && pipenv install --system --deploy

 # Install subset of dev dependencies needed for CI
-RUN pip install matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3
+RUN pip install matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3 coverage==5.1

 ENV PATH="/tmp/openpilot/external/bin:${PATH}"
 ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
@ -71,6 +71,8 @@ COPY ./flake8_openpilot.sh /tmp/openpilot/
 COPY ./pylint_openpilot.sh /tmp/openpilot/
 COPY ./.pylintrc /tmp/openpilot/

+COPY ./.coveragerc-app /tmp/openpilot/
+
 COPY ./pyextra /tmp/openpilot/pyextra
 COPY ./phonelibs /tmp/openpilot/phonelibs
 COPY ./external /tmp/openpilot/external
--- a/README.md
+++ b/README.md
@ -330,3 +330,5 @@ NO WARRANTY EXPRESSED OR IMPLIED.**

 [![Total alerts](https://img.shields.io/lgtm/alerts/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/alerts/)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:python)
+[![Language grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/commaai/openpilot.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/commaai/openpilot/context:cpp)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)
--- a/external/bin/MP4Box
+++ b/external/bin/MP4Box
@ -1 +0,0 @@
-../gpac/bin/MP4Box
--- a/external/bin/apktool
+++ b/external/bin/apktool
@ -1 +0,0 @@
-../apktool/apktool
--- a/external/bin/ipfs
+++ b/external/bin/ipfs
@ -1 +0,0 @@
-../ipfs/ipfs