diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index f906b8f65..9229fb176 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -1,70 +1,26 @@ FROM ubuntu:20.04 + ENV PYTHONUNBUFFERED 1 ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - bzip2 \ - ca-certificates \ - capnproto \ - clang \ - cmake \ - cppcheck \ - curl \ - ffmpeg \ - gcc-arm-none-eabi \ - git \ - iputils-ping \ - libarchive-dev \ - libbz2-dev \ - libcapnp-dev \ - libcurl4-openssl-dev \ - libeigen3-dev \ - libffi-dev \ - libgles2-mesa-dev \ - libglew-dev \ - libglib2.0-0 \ - liblzma-dev \ - libomp-dev \ - libopencv-dev \ - libqt5sql5-sqlite \ - libqt5svg5-dev \ - libsqlite3-dev \ - libssl-dev \ - libsystemd-dev \ - libusb-1.0-0-dev \ - libzmq3-dev \ - locales \ - ocl-icd-libopencl1 \ - ocl-icd-opencl-dev \ - opencl-headers \ - python-dev \ - qml-module-qtquick2 \ - qt5-default \ - qtlocation5-dev \ - qtmultimedia5-dev \ - qtpositioning5-dev \ - qtwebengine5-dev \ - sudo \ - valgrind \ - wget \ - && rm -rf /var/lib/apt/lists/* +RUN apt-get update && \ + apt-get install -y --no-install-recommends sudo tzdata locales && \ + rm -rf /var/lib/apt/lists/* RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en ENV LC_ALL en_US.UTF-8 -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" - -COPY Pipfile Pipfile.lock /tmp/ -RUN pyenv install 3.8.10 && \ - pyenv global 3.8.10 && \ - pyenv rehash && \ - pip install --no-cache-dir --upgrade pip==21.3.1 && \ - pip install --no-cache-dir pipenv==2021.5.29 && \ - cd /tmp && \ - pipenv install --system --deploy --dev --clear && \ +ENV PIPENV_SYSTEM=1 +COPY Pipfile Pipfile.lock .python-version update_requirements.sh /tmp/ +COPY tools/ubuntu_setup.sh /tmp/tools/ +RUN cd /tmp && \ + tools/ubuntu_setup.sh && \ + rm -rf /tmp/* && \ + rm -rf /var/lib/apt/lists/* && \ pip uninstall -y pipenv + +ENV PYENV_VERSION=3.8.10 +ENV PYENV_ROOT="/root/.pyenv" +ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH" diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index be5fc55a9..71ffe4a92 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -38,6 +38,7 @@ elif [[ $SHELL == "/bin/bash" ]]; then RC_FILE="$HOME/.bash_profile" fi +# TODO: get rid of this somehow # Build requirements for macOS # https://github.com/pyenv/pyenv/issues/1740 # https://github.com/pyca/cryptography/blob/main/docs/installation.rst @@ -52,7 +53,7 @@ export CPPFLAGS="$CPPFLAGS -I/usr/local/opt/openssl@1.1/include" export PATH="$PATH:/usr/local/opt/openssl@1.1/bin" export PATH="$PATH:/usr/local/bin" -# OpenPilot environment variables +# openpilot environment if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE @@ -61,15 +62,8 @@ if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then echo "Added openpilot_env to RC file: $RC_FILE" fi -# install python -PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version) -PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH -pyenv install -s ${PYENV_PYTHON_VERSION} -pyenv rehash -eval "$(pyenv init -)" - -pip install pipenv==2020.8.13 -pipenv install --dev --deploy +# install python dependencies +$ROOT/update_requirements.sh echo echo "---- FINISH OPENPILOT SETUP ----" diff --git a/tools/openpilot_env.sh b/tools/openpilot_env.sh index 3412c32d6..f8ce6901c 100755 --- a/tools/openpilot_env.sh +++ b/tools/openpilot_env.sh @@ -10,12 +10,10 @@ if [ -z "$OPENPILOT_ENV" ]; then # https://github.com/pyenv/pyenv/issues/1906 export PYENV_ROOT="$HOME/.pyenv" - unamestr=`uname` - if [[ "$unamestr" == 'Linux' ]]; then + if [[ "$(uname)" == 'Linux' ]]; then eval "$(pyenv init --path)" - eval "$(pyenv virtualenv-init -)" - elif [[ "$unamestr" == 'Darwin' ]]; then + elif [[ "$(uname)" == 'Darwin' ]]; then # msgq doesn't work on mac export ZMQ=1 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh index 0e1c0bc65..dd29c995e 100755 --- a/tools/ubuntu_setup.sh +++ b/tools/ubuntu_setup.sh @@ -1,8 +1,12 @@ -#!/bin/bash -e +#!/bin/bash + +set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ROOT="$(cd $DIR/../ && pwd)" +# NOTE: this is used in a docker build, so do not run any scripts here. + # Install packages present in all supported versions of Ubuntu function install_ubuntu_common_requirements() { sudo apt-get update @@ -70,7 +74,8 @@ function install_ubuntu_common_requirements() { libqt5svg5-dev \ libqt5x11extras5-dev \ libreadline-dev \ - libdw1 + libdw1 \ + valgrind } # Install Ubuntu 21.10 packages @@ -118,13 +123,8 @@ else fi -# install pyenv -if ! command -v "pyenv" > /dev/null 2>&1; then - curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -fi - -# in the openpilot repo -cd $ROOT +# install python dependencies +$ROOT/update_requirements.sh source ~/.bashrc if [ -z "$OPENPILOT_ENV" ]; then @@ -133,23 +133,6 @@ if [ -z "$OPENPILOT_ENV" ]; then echo "added openpilot_env to bashrc" fi -# do the rest of the git checkout -git lfs pull -git submodule init -git submodule update - -# install python -PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version) -PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH -pyenv install -s ${PYENV_PYTHON_VERSION} -pyenv rehash -eval "$(pyenv init -)" - -# **** in python env **** -pip install pip==21.3.1 -pip install pipenv==2021.5.29 -pipenv install --dev --deploy - echo echo "---- FINISH OPENPILOT SETUP ----" echo "Configure your active shell env by running:" diff --git a/update_requirements.sh b/update_requirements.sh index be7d53f98..fe35c4eec 100755 --- a/update_requirements.sh +++ b/update_requirements.sh @@ -1,17 +1,12 @@ #!/bin/bash -e -cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +cd $DIR -if ! command -v pyenv &> /dev/null; then - echo "please install pyenv ..." - echo "https://github.com/pyenv/pyenv-installer" - echo "example:" - echo "sudo apt-get update; sudo apt-get install --no-install-recommends make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev" - echo "curl https://pyenv.run | bash" - echo "echo 'export PYENV_ROOT=\"\$HOME/.pyenv\"' >> ~/.bashrc" - echo "echo 'export PATH=\"\$PYENV_ROOT/bin:\$PYENV_ROOT/shims:\$PATH\"' >> ~/.bashrc" - echo "exec \"\$SHELL\"" - exit 1 +if ! command -v "pyenv" > /dev/null 2>&1; then + echo "installing pyenv..." + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + export PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH fi export MAKEFLAGS="-j$(nproc)" @@ -19,7 +14,7 @@ export MAKEFLAGS="-j$(nproc)" PYENV_PYTHON_VERSION=$(cat .python-version) if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then echo "pyenv ${PYENV_PYTHON_VERSION} install ..." - CONFIGURE_OPTS=--enable-shared pyenv install -f ${PYENV_PYTHON_VERSION} + CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION} fi if ! command -v pipenv &> /dev/null; then @@ -29,24 +24,26 @@ fi echo "update pip" pip install pip==21.3.1 -pip install pipenv==2021.5.29 +pip install pipenv==2021.11.23 -echo "pip packages install ..." if [ -d "./xx" ]; then + export PIPENV_SYSTEM=1 export PIPENV_PIPFILE=./xx/Pipfile - pipenv install --system --dev --deploy - RUN="" -else - pipenv install --dev --deploy - RUN="pipenv run" fi -# update shims for newly installed executables (e.g. scons) +if [ -z "$PIPENV_SYSTEM" ]; then + RUN="pipenv run" +else + RUN="" +fi + +echo "pip packages install ..." +pipenv install --dev --deploy --clear pyenv rehash -echo "precommit install ..." -$RUN pre-commit install - -# for internal comma repos -[ -d "./xx" ] && (cd xx && $RUN pre-commit install) -[ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install) +if [ -f "$DIR/.pre-commit-config.yaml" ]; then + echo "precommit install ..." + $RUN pre-commit install + [ -d "./xx" ] && (cd xx && $RUN pre-commit install) + [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install) +fi