Fast CI (#1537)
* don't compile in docker build stage * disable that for now * do we need that? * upload artifact already compresses * that wasn't faster This reverts commit 2221e13c71b5cc693dd3bebfa4061d4ab6821437. * try that * don't think that's used * this too * check this * less layers * that's not in the dockerfile anymore * system * even less layers and clear pipenv cache * no inline comment i guess * uninstall pipenv after we're done with it * separate build job adds too much overhead * quotes * clean that up * increase timeout * cleanupalbatross
parent
8d46c0fa8a
commit
45ee131a44
|
@ -1,37 +1,21 @@
|
||||||
name: Openpilot Tests
|
name: openpilot tests
|
||||||
on: [push, pull_request]
|
on: [push, pull_request]
|
||||||
|
|
||||||
env:
|
env:
|
||||||
RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
|
RUN: docker run --shm-size 1G --rm tmppilot /bin/sh -c
|
||||||
PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
|
PERSIST: docker run --shm-size 1G --name tmppilot tmppilot /bin/sh -c
|
||||||
LOAD: docker load -i tmppilot.tar.gz/tmppilot.tar.gz
|
|
||||||
CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
|
CI_RUN: docker run -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID --rm tmppilotci /bin/bash -c
|
||||||
UNIT_TEST: coverage run --append -m unittest discover
|
UNIT_TEST: coverage run --append -m unittest discover
|
||||||
|
BUILD: |
|
||||||
|
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
|
||||||
|
docker pull docker.io/commaai/openpilot:latest || true
|
||||||
|
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
|
||||||
name: build
|
|
||||||
runs-on: ubuntu-16.04
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v2
|
|
||||||
with:
|
|
||||||
submodules: true
|
|
||||||
- name: Build docker image
|
|
||||||
run: |
|
|
||||||
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
|
|
||||||
docker pull docker.io/commaai/openpilot:latest || true
|
|
||||||
|
|
||||||
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
|
|
||||||
docker save tmppilot:latest | gzip > tmppilot.tar.gz
|
|
||||||
- uses: actions/upload-artifact@v2
|
|
||||||
with:
|
|
||||||
name: tmppilot.tar.gz
|
|
||||||
path: tmppilot.tar.gz
|
|
||||||
|
|
||||||
build_release:
|
build_release:
|
||||||
name: build release
|
name: build release
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
timeout-minutes: 30
|
timeout-minutes: 50
|
||||||
env:
|
env:
|
||||||
TEST_DIR: tmppilot
|
TEST_DIR: tmppilot
|
||||||
steps:
|
steps:
|
||||||
|
@ -56,41 +40,37 @@ jobs:
|
||||||
.coveragerc-app $TEST_DIR
|
.coveragerc-app $TEST_DIR
|
||||||
cd $TEST_DIR
|
cd $TEST_DIR
|
||||||
mkdir pyextra laika laika_repo tools release
|
mkdir pyextra laika laika_repo tools release
|
||||||
- name: Build
|
- name: Build Docker image
|
||||||
run: |
|
run: cd $TEST_DIR && eval "$BUILD"
|
||||||
cd $TEST_DIR
|
- name: Build openpilot
|
||||||
docker pull $(grep -ioP '(?<=^from)\s+\S+' Dockerfile.openpilot) || true
|
run: $RUN "cd /tmp/openpilot && scons -j$(nproc)"
|
||||||
docker pull docker.io/commaai/openpilot:latest || true
|
|
||||||
docker build --cache-from docker.io/commaai/openpilot:latest -t tmppilot -f Dockerfile.openpilot .
|
|
||||||
|
|
||||||
push:
|
docker_push:
|
||||||
name: push
|
name: docker push
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
|
||||||
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
|
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
|
||||||
|
needs: linter # hack to ensure slow tests run first since this and linter are fast
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: Login to dockerhub
|
- name: Push to dockerhub
|
||||||
run: docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }}
|
run: |
|
||||||
- name: Tag image
|
docker login -u wmelching -p ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
run: docker tag tmppilot docker.io/commaai/openpilot:latest
|
docker tag tmppilot docker.io/commaai/openpilot:latest
|
||||||
- name: Push image
|
docker push docker.io/commaai/openpilot:latest
|
||||||
run: docker push docker.io/commaai/openpilot:latest
|
|
||||||
|
|
||||||
linter:
|
linter:
|
||||||
name: linter
|
name: linter
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: flake8
|
- name: flake8
|
||||||
run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
|
run: $RUN "cd /tmp/openpilot/ && ./flake8_openpilot.sh"
|
||||||
- name: pylint
|
- name: pylint
|
||||||
|
@ -99,16 +79,16 @@ jobs:
|
||||||
unit_tests:
|
unit_tests:
|
||||||
name: unit tests
|
name: unit tests
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: Run unit tests
|
- name: Run unit tests
|
||||||
run: |
|
run: |
|
||||||
$PERSIST "cd /tmp/openpilot && \
|
$PERSIST "cd /tmp/openpilot && \
|
||||||
|
scons -j$(nproc) && \
|
||||||
coverage run selfdrive/test/test_fingerprints.py && \
|
coverage run selfdrive/test/test_fingerprints.py && \
|
||||||
$UNIT_TEST common && \
|
$UNIT_TEST common && \
|
||||||
$UNIT_TEST opendbc/can && \
|
$UNIT_TEST opendbc/can && \
|
||||||
|
@ -127,17 +107,18 @@ jobs:
|
||||||
process_replay:
|
process_replay:
|
||||||
name: process replay
|
name: process replay
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
timeout-minutes: 50
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: Run replay
|
- name: Run replay
|
||||||
run: |
|
run: |
|
||||||
$PERSIST "cd /tmp/openpilot && CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
|
$PERSIST "cd /tmp/openpilot && \
|
||||||
|
scons -j$(nproc) && \
|
||||||
|
CI=1 coverage run selfdrive/test/process_replay/test_processes.py"
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
run: |
|
run: |
|
||||||
docker commit tmppilot tmppilotci
|
docker commit tmppilot tmppilotci
|
||||||
|
@ -158,17 +139,20 @@ jobs:
|
||||||
test_longitudinal:
|
test_longitudinal:
|
||||||
name: longitudinal
|
name: longitudinal
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
timeout-minutes: 50
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: Test longitudinal
|
- name: Test longitudinal
|
||||||
run: |
|
run: |
|
||||||
$PERSIST "cd /tmp/openpilot/selfdrive/test/longitudinal_maneuvers && OPTEST=1 ./test_longitudinal.py"
|
$PERSIST "mkdir -p /tmp/openpilot/selfdrive/test/out && \
|
||||||
|
cd /tmp/openpilot/ && \
|
||||||
|
scons -j$(nproc) && \
|
||||||
|
cd selfdrive/test/longitudinal_maneuvers && \
|
||||||
|
OPTEST=1 ./test_longitudinal.py"
|
||||||
- name: Copy artifacts
|
- name: Copy artifacts
|
||||||
if: always()
|
if: always()
|
||||||
run: |
|
run: |
|
||||||
|
@ -183,17 +167,20 @@ jobs:
|
||||||
test_car_models:
|
test_car_models:
|
||||||
name: test car models
|
name: test car models
|
||||||
runs-on: ubuntu-16.04
|
runs-on: ubuntu-16.04
|
||||||
needs: build
|
timeout-minutes: 50
|
||||||
timeout-minutes: 30
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v1
|
- uses: actions/checkout@v2
|
||||||
with:
|
with:
|
||||||
name: tmppilot.tar.gz
|
submodules: true
|
||||||
- name: Load image
|
- name: Build Docker image
|
||||||
run: $LOAD
|
run: eval "$BUILD"
|
||||||
- name: Test car models
|
- name: Test car models
|
||||||
run: |
|
run: |
|
||||||
$PERSIST "mkdir -p /data/params && cd /tmp/openpilot && coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && coverage combine"
|
$PERSIST "mkdir -p /data/params && \
|
||||||
|
cd /tmp/openpilot && \
|
||||||
|
scons -j$(nproc) && \
|
||||||
|
coverage run --parallel-mode --concurrency=multiprocessing --rcfile=./.coveragerc-app selfdrive/test/test_car_models.py && \
|
||||||
|
coverage combine"
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
run: |
|
run: |
|
||||||
docker commit tmppilot tmppilotci
|
docker commit tmppilot tmppilotci
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
FROM ubuntu:16.04
|
FROM ubuntu:16.04
|
||||||
ENV PYTHONUNBUFFERED 1
|
ENV PYTHONUNBUFFERED 1
|
||||||
|
ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
autoconf \
|
autoconf \
|
||||||
|
@ -22,12 +23,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
libglfw3-dev \
|
libglfw3-dev \
|
||||||
libglib2.0-0 \
|
libglib2.0-0 \
|
||||||
liblzma-dev \
|
liblzma-dev \
|
||||||
libmysqlclient-dev \
|
|
||||||
libomp-dev \
|
libomp-dev \
|
||||||
libopencv-dev \
|
libopencv-dev \
|
||||||
libssl-dev \
|
libssl-dev \
|
||||||
libsqlite3-dev \
|
libsqlite3-dev \
|
||||||
libtool \
|
|
||||||
libusb-1.0-0-dev \
|
libusb-1.0-0-dev \
|
||||||
libczmq-dev \
|
libczmq-dev \
|
||||||
libzmq3-dev \
|
libzmq3-dev \
|
||||||
|
@ -39,33 +38,49 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
python-pip \
|
python-pip \
|
||||||
sudo \
|
sudo \
|
||||||
wget \
|
wget \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
|
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
|
||||||
ENV LANG en_US.UTF-8
|
ENV LANG en_US.UTF-8
|
||||||
ENV LANGUAGE en_US:en
|
ENV LANGUAGE en_US:en
|
||||||
ENV LC_ALL en_US.UTF-8
|
ENV LC_ALL en_US.UTF-8
|
||||||
|
|
||||||
|
# Install python dependencies
|
||||||
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
||||||
|
|
||||||
ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
|
ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
|
||||||
RUN pyenv install 3.8.2 && pyenv global 3.8.2
|
|
||||||
RUN pyenv rehash
|
|
||||||
|
|
||||||
RUN pip install --no-cache-dir pipenv==2018.11.26
|
|
||||||
|
|
||||||
COPY Pipfile Pipfile.lock /tmp/
|
COPY Pipfile Pipfile.lock /tmp/
|
||||||
|
RUN pyenv install 3.8.2 && \
|
||||||
RUN cd /tmp && pipenv install --system --deploy
|
pyenv global 3.8.2 && \
|
||||||
|
pyenv rehash && \
|
||||||
# Install subset of dev dependencies needed for CI
|
pip install --no-cache-dir pipenv==2018.11.26 && \
|
||||||
RUN pip install --no-cache-dir matplotlib==3.1.1 dictdiffer==0.8.0 fastcluster==1.1.25 aenum==2.2.1 lru-dict==1.1.6 scipy==1.4.1 tenacity==5.1.1 azure-common==1.1.23 azure-nspkg==3.0.2 azure-storage-blob==2.1.0 azure-storage-common==2.1.0 azure-storage-nspkg==3.1.0 pycurl==7.43.0.3 coverage==5.1
|
cd /tmp && \
|
||||||
|
pipenv install --system --deploy --clear && \
|
||||||
ENV PYTHONPATH /tmp/openpilot:${PYTHONPATH}
|
pip uninstall -y pipenv && \
|
||||||
|
pip install --no-cache-dir \
|
||||||
|
matplotlib==3.1.1 \
|
||||||
|
dictdiffer==0.8.0 \
|
||||||
|
fastcluster==1.1.25 \
|
||||||
|
aenum==2.2.1 \
|
||||||
|
lru-dict==1.1.6 \
|
||||||
|
scipy==1.4.1 \
|
||||||
|
tenacity==5.1.1 \
|
||||||
|
azure-common==1.1.23 \
|
||||||
|
azure-nspkg==3.0.2 \
|
||||||
|
azure-storage-blob==2.1.0 \
|
||||||
|
azure-storage-common==2.1.0 \
|
||||||
|
azure-storage-nspkg==3.1.0 \
|
||||||
|
pycurl==7.43.0.3 \
|
||||||
|
coverage==5.1
|
||||||
|
|
||||||
RUN mkdir -p /tmp/openpilot
|
RUN mkdir -p /tmp/openpilot
|
||||||
|
|
||||||
COPY ./flake8_openpilot.sh ./pylint_openpilot.sh ./.pylintrc ./.coveragerc-app /tmp/openpilot/
|
COPY SConstruct \
|
||||||
|
flake8_openpilot.sh \
|
||||||
|
pylint_openpilot.sh \
|
||||||
|
.pylintrc \
|
||||||
|
.coveragerc-app \
|
||||||
|
/tmp/openpilot/
|
||||||
|
|
||||||
COPY ./pyextra /tmp/openpilot/pyextra
|
COPY ./pyextra /tmp/openpilot/pyextra
|
||||||
COPY ./phonelibs /tmp/openpilot/phonelibs
|
COPY ./phonelibs /tmp/openpilot/phonelibs
|
||||||
|
@ -79,8 +94,3 @@ COPY ./opendbc /tmp/openpilot/opendbc
|
||||||
COPY ./cereal /tmp/openpilot/cereal
|
COPY ./cereal /tmp/openpilot/cereal
|
||||||
COPY ./panda /tmp/openpilot/panda
|
COPY ./panda /tmp/openpilot/panda
|
||||||
COPY ./selfdrive /tmp/openpilot/selfdrive
|
COPY ./selfdrive /tmp/openpilot/selfdrive
|
||||||
|
|
||||||
COPY SConstruct /tmp/openpilot/SConstruct
|
|
||||||
|
|
||||||
RUN mkdir -p /tmp/openpilot/selfdrive/test/out
|
|
||||||
RUN cd /tmp/openpilot && scons -j$(nproc)
|
|
||||||
|
|
Loading…
Reference in New Issue