CI: use ubuntu_setup.sh (#23349)
* CI: use ubuntu_setup.sh * works? * need to fix paths * set paths * no batman for now * pyenv version * rehash after install packages * unify python install * revert that * fix pyenv path * no pipenv * flip that * fix rehash * skip that * revert pandapull/23360/head
parent
9992ea1ae8
commit
066e3e3626
|
@ -1,70 +1,26 @@
|
||||||
FROM ubuntu:20.04
|
FROM ubuntu:20.04
|
||||||
|
|
||||||
ENV PYTHONUNBUFFERED 1
|
ENV PYTHONUNBUFFERED 1
|
||||||
|
|
||||||
ENV DEBIAN_FRONTEND=noninteractive
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && \
|
||||||
autoconf \
|
apt-get install -y --no-install-recommends sudo tzdata locales && \
|
||||||
build-essential \
|
rm -rf /var/lib/apt/lists/*
|
||||||
bzip2 \
|
|
||||||
ca-certificates \
|
|
||||||
capnproto \
|
|
||||||
clang \
|
|
||||||
cmake \
|
|
||||||
cppcheck \
|
|
||||||
curl \
|
|
||||||
ffmpeg \
|
|
||||||
gcc-arm-none-eabi \
|
|
||||||
git \
|
|
||||||
iputils-ping \
|
|
||||||
libarchive-dev \
|
|
||||||
libbz2-dev \
|
|
||||||
libcapnp-dev \
|
|
||||||
libcurl4-openssl-dev \
|
|
||||||
libeigen3-dev \
|
|
||||||
libffi-dev \
|
|
||||||
libgles2-mesa-dev \
|
|
||||||
libglew-dev \
|
|
||||||
libglib2.0-0 \
|
|
||||||
liblzma-dev \
|
|
||||||
libomp-dev \
|
|
||||||
libopencv-dev \
|
|
||||||
libqt5sql5-sqlite \
|
|
||||||
libqt5svg5-dev \
|
|
||||||
libsqlite3-dev \
|
|
||||||
libssl-dev \
|
|
||||||
libsystemd-dev \
|
|
||||||
libusb-1.0-0-dev \
|
|
||||||
libzmq3-dev \
|
|
||||||
locales \
|
|
||||||
ocl-icd-libopencl1 \
|
|
||||||
ocl-icd-opencl-dev \
|
|
||||||
opencl-headers \
|
|
||||||
python-dev \
|
|
||||||
qml-module-qtquick2 \
|
|
||||||
qt5-default \
|
|
||||||
qtlocation5-dev \
|
|
||||||
qtmultimedia5-dev \
|
|
||||||
qtpositioning5-dev \
|
|
||||||
qtwebengine5-dev \
|
|
||||||
sudo \
|
|
||||||
valgrind \
|
|
||||||
wget \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
|
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
|
||||||
ENV LANG en_US.UTF-8
|
ENV LANG en_US.UTF-8
|
||||||
ENV LANGUAGE en_US:en
|
ENV LANGUAGE en_US:en
|
||||||
ENV LC_ALL en_US.UTF-8
|
ENV LC_ALL en_US.UTF-8
|
||||||
|
|
||||||
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
ENV PIPENV_SYSTEM=1
|
||||||
ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
|
COPY Pipfile Pipfile.lock .python-version update_requirements.sh /tmp/
|
||||||
|
COPY tools/ubuntu_setup.sh /tmp/tools/
|
||||||
COPY Pipfile Pipfile.lock /tmp/
|
RUN cd /tmp && \
|
||||||
RUN pyenv install 3.8.10 && \
|
tools/ubuntu_setup.sh && \
|
||||||
pyenv global 3.8.10 && \
|
rm -rf /tmp/* && \
|
||||||
pyenv rehash && \
|
rm -rf /var/lib/apt/lists/* && \
|
||||||
pip install --no-cache-dir --upgrade pip==21.3.1 && \
|
|
||||||
pip install --no-cache-dir pipenv==2021.5.29 && \
|
|
||||||
cd /tmp && \
|
|
||||||
pipenv install --system --deploy --dev --clear && \
|
|
||||||
pip uninstall -y pipenv
|
pip uninstall -y pipenv
|
||||||
|
|
||||||
|
ENV PYENV_VERSION=3.8.10
|
||||||
|
ENV PYENV_ROOT="/root/.pyenv"
|
||||||
|
ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
|
||||||
|
|
|
@ -38,6 +38,7 @@ elif [[ $SHELL == "/bin/bash" ]]; then
|
||||||
RC_FILE="$HOME/.bash_profile"
|
RC_FILE="$HOME/.bash_profile"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# TODO: get rid of this somehow
|
||||||
# Build requirements for macOS
|
# Build requirements for macOS
|
||||||
# https://github.com/pyenv/pyenv/issues/1740
|
# https://github.com/pyenv/pyenv/issues/1740
|
||||||
# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
|
# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
|
||||||
|
@ -52,7 +53,7 @@ export CPPFLAGS="$CPPFLAGS -I/usr/local/opt/openssl@1.1/include"
|
||||||
export PATH="$PATH:/usr/local/opt/openssl@1.1/bin"
|
export PATH="$PATH:/usr/local/opt/openssl@1.1/bin"
|
||||||
export PATH="$PATH:/usr/local/bin"
|
export PATH="$PATH:/usr/local/bin"
|
||||||
|
|
||||||
# OpenPilot environment variables
|
# openpilot environment
|
||||||
if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
|
if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
|
||||||
echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
|
echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
|
||||||
echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
|
echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
|
||||||
|
@ -61,15 +62,8 @@ if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
|
||||||
echo "Added openpilot_env to RC file: $RC_FILE"
|
echo "Added openpilot_env to RC file: $RC_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# install python
|
# install python dependencies
|
||||||
PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
|
$ROOT/update_requirements.sh
|
||||||
PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
|
|
||||||
pyenv install -s ${PYENV_PYTHON_VERSION}
|
|
||||||
pyenv rehash
|
|
||||||
eval "$(pyenv init -)"
|
|
||||||
|
|
||||||
pip install pipenv==2020.8.13
|
|
||||||
pipenv install --dev --deploy
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "---- FINISH OPENPILOT SETUP ----"
|
echo "---- FINISH OPENPILOT SETUP ----"
|
||||||
|
|
|
@ -10,12 +10,10 @@ if [ -z "$OPENPILOT_ENV" ]; then
|
||||||
# https://github.com/pyenv/pyenv/issues/1906
|
# https://github.com/pyenv/pyenv/issues/1906
|
||||||
export PYENV_ROOT="$HOME/.pyenv"
|
export PYENV_ROOT="$HOME/.pyenv"
|
||||||
|
|
||||||
unamestr=`uname`
|
if [[ "$(uname)" == 'Linux' ]]; then
|
||||||
if [[ "$unamestr" == 'Linux' ]]; then
|
|
||||||
eval "$(pyenv init --path)"
|
eval "$(pyenv init --path)"
|
||||||
|
|
||||||
eval "$(pyenv virtualenv-init -)"
|
eval "$(pyenv virtualenv-init -)"
|
||||||
elif [[ "$unamestr" == 'Darwin' ]]; then
|
elif [[ "$(uname)" == 'Darwin' ]]; then
|
||||||
# msgq doesn't work on mac
|
# msgq doesn't work on mac
|
||||||
export ZMQ=1
|
export ZMQ=1
|
||||||
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
|
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
|
||||||
|
|
|
@ -1,8 +1,12 @@
|
||||||
#!/bin/bash -e
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||||
ROOT="$(cd $DIR/../ && pwd)"
|
ROOT="$(cd $DIR/../ && pwd)"
|
||||||
|
|
||||||
|
# NOTE: this is used in a docker build, so do not run any scripts here.
|
||||||
|
|
||||||
# Install packages present in all supported versions of Ubuntu
|
# Install packages present in all supported versions of Ubuntu
|
||||||
function install_ubuntu_common_requirements() {
|
function install_ubuntu_common_requirements() {
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
|
@ -70,7 +74,8 @@ function install_ubuntu_common_requirements() {
|
||||||
libqt5svg5-dev \
|
libqt5svg5-dev \
|
||||||
libqt5x11extras5-dev \
|
libqt5x11extras5-dev \
|
||||||
libreadline-dev \
|
libreadline-dev \
|
||||||
libdw1
|
libdw1 \
|
||||||
|
valgrind
|
||||||
}
|
}
|
||||||
|
|
||||||
# Install Ubuntu 21.10 packages
|
# Install Ubuntu 21.10 packages
|
||||||
|
@ -118,13 +123,8 @@ else
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
# install pyenv
|
# install python dependencies
|
||||||
if ! command -v "pyenv" > /dev/null 2>&1; then
|
$ROOT/update_requirements.sh
|
||||||
curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
|
||||||
fi
|
|
||||||
|
|
||||||
# in the openpilot repo
|
|
||||||
cd $ROOT
|
|
||||||
|
|
||||||
source ~/.bashrc
|
source ~/.bashrc
|
||||||
if [ -z "$OPENPILOT_ENV" ]; then
|
if [ -z "$OPENPILOT_ENV" ]; then
|
||||||
|
@ -133,23 +133,6 @@ if [ -z "$OPENPILOT_ENV" ]; then
|
||||||
echo "added openpilot_env to bashrc"
|
echo "added openpilot_env to bashrc"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# do the rest of the git checkout
|
|
||||||
git lfs pull
|
|
||||||
git submodule init
|
|
||||||
git submodule update
|
|
||||||
|
|
||||||
# install python
|
|
||||||
PYENV_PYTHON_VERSION=$(cat $ROOT/.python-version)
|
|
||||||
PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
|
|
||||||
pyenv install -s ${PYENV_PYTHON_VERSION}
|
|
||||||
pyenv rehash
|
|
||||||
eval "$(pyenv init -)"
|
|
||||||
|
|
||||||
# **** in python env ****
|
|
||||||
pip install pip==21.3.1
|
|
||||||
pip install pipenv==2021.5.29
|
|
||||||
pipenv install --dev --deploy
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "---- FINISH OPENPILOT SETUP ----"
|
echo "---- FINISH OPENPILOT SETUP ----"
|
||||||
echo "Configure your active shell env by running:"
|
echo "Configure your active shell env by running:"
|
||||||
|
|
|
@ -1,17 +1,12 @@
|
||||||
#!/bin/bash -e
|
#!/bin/bash -e
|
||||||
|
|
||||||
cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
||||||
|
cd $DIR
|
||||||
|
|
||||||
if ! command -v pyenv &> /dev/null; then
|
if ! command -v "pyenv" > /dev/null 2>&1; then
|
||||||
echo "please install pyenv ..."
|
echo "installing pyenv..."
|
||||||
echo "https://github.com/pyenv/pyenv-installer"
|
curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
||||||
echo "example:"
|
export PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH
|
||||||
echo "sudo apt-get update; sudo apt-get install --no-install-recommends make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev"
|
|
||||||
echo "curl https://pyenv.run | bash"
|
|
||||||
echo "echo 'export PYENV_ROOT=\"\$HOME/.pyenv\"' >> ~/.bashrc"
|
|
||||||
echo "echo 'export PATH=\"\$PYENV_ROOT/bin:\$PYENV_ROOT/shims:\$PATH\"' >> ~/.bashrc"
|
|
||||||
echo "exec \"\$SHELL\""
|
|
||||||
exit 1
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
export MAKEFLAGS="-j$(nproc)"
|
export MAKEFLAGS="-j$(nproc)"
|
||||||
|
@ -19,7 +14,7 @@ export MAKEFLAGS="-j$(nproc)"
|
||||||
PYENV_PYTHON_VERSION=$(cat .python-version)
|
PYENV_PYTHON_VERSION=$(cat .python-version)
|
||||||
if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
|
if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
|
||||||
echo "pyenv ${PYENV_PYTHON_VERSION} install ..."
|
echo "pyenv ${PYENV_PYTHON_VERSION} install ..."
|
||||||
CONFIGURE_OPTS=--enable-shared pyenv install -f ${PYENV_PYTHON_VERSION}
|
CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if ! command -v pipenv &> /dev/null; then
|
if ! command -v pipenv &> /dev/null; then
|
||||||
|
@ -29,24 +24,26 @@ fi
|
||||||
|
|
||||||
echo "update pip"
|
echo "update pip"
|
||||||
pip install pip==21.3.1
|
pip install pip==21.3.1
|
||||||
pip install pipenv==2021.5.29
|
pip install pipenv==2021.11.23
|
||||||
|
|
||||||
echo "pip packages install ..."
|
|
||||||
if [ -d "./xx" ]; then
|
if [ -d "./xx" ]; then
|
||||||
|
export PIPENV_SYSTEM=1
|
||||||
export PIPENV_PIPFILE=./xx/Pipfile
|
export PIPENV_PIPFILE=./xx/Pipfile
|
||||||
pipenv install --system --dev --deploy
|
|
||||||
RUN=""
|
|
||||||
else
|
|
||||||
pipenv install --dev --deploy
|
|
||||||
RUN="pipenv run"
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# update shims for newly installed executables (e.g. scons)
|
if [ -z "$PIPENV_SYSTEM" ]; then
|
||||||
|
RUN="pipenv run"
|
||||||
|
else
|
||||||
|
RUN=""
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "pip packages install ..."
|
||||||
|
pipenv install --dev --deploy --clear
|
||||||
pyenv rehash
|
pyenv rehash
|
||||||
|
|
||||||
echo "precommit install ..."
|
if [ -f "$DIR/.pre-commit-config.yaml" ]; then
|
||||||
$RUN pre-commit install
|
echo "precommit install ..."
|
||||||
|
$RUN pre-commit install
|
||||||
# for internal comma repos
|
[ -d "./xx" ] && (cd xx && $RUN pre-commit install)
|
||||||
[ -d "./xx" ] && (cd xx && $RUN pre-commit install)
|
[ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
|
||||||
[ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
|
fi
|
||||||
|
|
Loading…
Reference in New Issue