CI: use ubuntu_setup.sh (#23349)

* CI: use ubuntu_setup.sh

* works?

* need to fix paths

* set paths

* no batman for now

* pyenv version

* rehash after install packages

* unify python install

* revert that

* fix pyenv path

* no pipenv

* flip that

* fix rehash

* skip that

* revert panda

Dockerfile.openpilot_base

@@ -1,70 +1,26 @@
 FROM ubuntu:20.04
+
 ENV PYTHONUNBUFFERED 1
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    autoconf \
-    build-essential \
-    bzip2 \
-    ca-certificates \
-    capnproto \
-    clang \
-    cmake \
-    cppcheck \
-    curl \
-    ffmpeg \
-    gcc-arm-none-eabi \
-    git \
-    iputils-ping \
-    libarchive-dev \
-    libbz2-dev \
-    libcapnp-dev \
-    libcurl4-openssl-dev \
-    libeigen3-dev \
-    libffi-dev \
-    libgles2-mesa-dev \
-    libglew-dev \
-    libglib2.0-0 \
-    liblzma-dev \
-    libomp-dev \
-    libopencv-dev \
-    libqt5sql5-sqlite \
-    libqt5svg5-dev \
-    libsqlite3-dev \
-    libssl-dev \
-    libsystemd-dev \
-    libusb-1.0-0-dev \
-    libzmq3-dev \
-    locales \
-    ocl-icd-libopencl1 \
-    ocl-icd-opencl-dev \
-    opencl-headers \
-    python-dev \
-    qml-module-qtquick2 \
-    qt5-default \
-    qtlocation5-dev \
-    qtmultimedia5-dev \
-    qtpositioning5-dev \
-    qtwebengine5-dev \
-    sudo \
-    valgrind \
-    wget \
-  && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales && \
+    rm -rf /var/lib/apt/lists/*
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
-
-COPY Pipfile Pipfile.lock /tmp/
-RUN pyenv install 3.8.10 && \
-    pyenv global 3.8.10 && \
-    pyenv rehash && \
-    pip install --no-cache-dir --upgrade pip==21.3.1 && \
-    pip install --no-cache-dir pipenv==2021.5.29 && \
-    cd /tmp && \
-    pipenv install --system --deploy --dev --clear && \
+ENV PIPENV_SYSTEM=1
+COPY Pipfile Pipfile.lock .python-version update_requirements.sh /tmp/
+COPY tools/ubuntu_setup.sh /tmp/tools/
+RUN cd /tmp && \
+    tools/ubuntu_setup.sh && \
+    rm -rf /tmp/* && \
+    rm -rf /var/lib/apt/lists/* && \
     pip uninstall -y pipenv
+
+ENV PYENV_VERSION=3.8.10
+ENV PYENV_ROOT="/root/.pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"

tools/mac_setup.sh

@@ -38,6 +38,7 @@ elif [[ $SHELL == "/bin/bash" ]]; then
   RC_FILE="$HOME/.bash_profile"
 fi
 
+# TODO: get rid of this somehow
 # Build requirements for macOS
 # https://github.com/pyenv/pyenv/issues/1740
 # https://github.com/pyca/cryptography/blob/main/docs/installation.rst
@@ -52,7 +53,7 @@ export CPPFLAGS="$CPPFLAGS -I/usr/local/opt/openssl@1.1/include"
 export PATH="$PATH:/usr/local/opt/openssl@1.1/bin"
 export PATH="$PATH:/usr/local/bin"
 
-# OpenPilot environment variables
+# openpilot environment
 if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
   echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
   echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
@@ -61,15 +62,8 @@ if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
   echo "Added openpilot_env to RC file: $RC_FILE"
 fi
 
-# install python
-PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
-PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
-pyenv install -s ${PYENV_PYTHON_VERSION}
-pyenv rehash
-eval "$(pyenv init -)"
-
-pip install pipenv==2020.8.13
-pipenv install --dev --deploy
+# install python dependencies
+$ROOT/update_requirements.sh
 
 echo
 echo "----   FINISH OPENPILOT SETUP   ----"

tools/openpilot_env.sh

@@ -10,12 +10,10 @@ if [ -z "$OPENPILOT_ENV" ]; then
   # https://github.com/pyenv/pyenv/issues/1906
   export PYENV_ROOT="$HOME/.pyenv"
 
-  unamestr=`uname`
-  if [[ "$unamestr" == 'Linux' ]]; then
+  if [[ "$(uname)" == 'Linux' ]]; then
     eval "$(pyenv init --path)"
-
     eval "$(pyenv virtualenv-init -)"
-  elif [[ "$unamestr" == 'Darwin' ]]; then
+  elif [[ "$(uname)" == 'Darwin' ]]; then
     # msgq doesn't work on mac
     export ZMQ=1
     export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES

tools/ubuntu_setup.sh

@@ -1,8 +1,12 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT="$(cd $DIR/../ && pwd)"
 
+# NOTE: this is used in a docker build, so do not run any scripts here.
+
 # Install packages present in all supported versions of Ubuntu
 function install_ubuntu_common_requirements() {
   sudo apt-get update
@@ -70,7 +74,8 @@ function install_ubuntu_common_requirements() {
     libqt5svg5-dev \
     libqt5x11extras5-dev \
     libreadline-dev \
-    libdw1
+    libdw1 \
+    valgrind
 }
 
 # Install Ubuntu 21.10 packages
@@ -118,13 +123,8 @@ else
 fi
 
 
-# install pyenv
-if ! command -v "pyenv" > /dev/null 2>&1; then
-  curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-fi
-
-# in the openpilot repo
-cd $ROOT
+# install python dependencies
+$ROOT/update_requirements.sh
 
 source ~/.bashrc
 if [ -z "$OPENPILOT_ENV" ]; then
@@ -133,23 +133,6 @@ if [ -z "$OPENPILOT_ENV" ]; then
   echo "added openpilot_env to bashrc"
 fi
 
-# do the rest of the git checkout
-git lfs pull
-git submodule init
-git submodule update
-
-# install python
-PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
-PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
-pyenv install -s ${PYENV_PYTHON_VERSION}
-pyenv rehash
-eval "$(pyenv init -)"
-
-# **** in python env ****
-pip install pip==21.3.1
-pip install pipenv==2021.5.29
-pipenv install --dev --deploy
-
 echo
 echo "----   FINISH OPENPILOT SETUP   ----"
 echo "Configure your active shell env by running:"

update_requirements.sh

@@ -1,17 +1,12 @@
 #!/bin/bash -e
 
-cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+cd $DIR
 
-if ! command -v pyenv &> /dev/null; then
-  echo "please install pyenv ..."
-  echo "https://github.com/pyenv/pyenv-installer"
-  echo "example:"
-  echo "sudo apt-get update; sudo apt-get install --no-install-recommends make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev"
-  echo "curl https://pyenv.run | bash"
-  echo "echo 'export PYENV_ROOT=\"\$HOME/.pyenv\"' >> ~/.bashrc"
-  echo "echo 'export PATH=\"\$PYENV_ROOT/bin:\$PYENV_ROOT/shims:\$PATH\"' >> ~/.bashrc"
-  echo "exec \"\$SHELL\""
-  exit 1
+if ! command -v "pyenv" > /dev/null 2>&1; then
+  echo "installing pyenv..."
+  curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+  export PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
 fi
 
 export MAKEFLAGS="-j$(nproc)"
@@ -19,7 +14,7 @@ export MAKEFLAGS="-j$(nproc)"
 PYENV_PYTHON_VERSION=$(cat .python-version)
 if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
   echo "pyenv ${PYENV_PYTHON_VERSION} install ..."
-  CONFIGURE_OPTS=--enable-shared pyenv install -f ${PYENV_PYTHON_VERSION}
+  CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
 fi
 
 if ! command -v pipenv &> /dev/null; then
@@ -29,24 +24,26 @@ fi
 
 echo "update pip"
 pip install pip==21.3.1
-pip install pipenv==2021.5.29
+pip install pipenv==2021.11.23
 
-echo "pip packages install ..."
 if [ -d "./xx" ]; then
+  export PIPENV_SYSTEM=1
   export PIPENV_PIPFILE=./xx/Pipfile
-  pipenv install --system --dev --deploy
-  RUN=""
-else
-  pipenv install --dev --deploy
-  RUN="pipenv run"
 fi
 
-# update shims for newly installed executables (e.g. scons)
+if [ -z "$PIPENV_SYSTEM" ]; then
+  RUN="pipenv run"
+else
+  RUN=""
+fi
+
+echo "pip packages install ..."
+pipenv install --dev --deploy --clear
 pyenv rehash
 
-echo "precommit install ..."
-$RUN pre-commit install
-
-# for internal comma repos
-[ -d "./xx" ] && (cd xx && $RUN pre-commit install)
-[ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
+if [ -f "$DIR/.pre-commit-config.yaml" ]; then
+  echo "precommit install ..."
+  $RUN pre-commit install
+  [ -d "./xx" ] && (cd xx && $RUN pre-commit install)
+  [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
+fi