diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 1bee59e45..f04eaad86 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -69,52 +69,62 @@ jobs: rm -rf /tmp/scons_cache/* && \ scons -j$(nproc) --cache-populate" - #build_mac: - # name: build macos - # runs-on: macos-10.15 - # timeout-minutes: 60 - # steps: - # - uses: actions/checkout@v2 - # with: - # submodules: true - # - name: Determine pre-existing Homebrew packages - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # echo 'EXISTING_CELLAR<> $GITHUB_ENV - # ls -1 /usr/local/Cellar >> $GITHUB_ENV - # echo 'EOF' >> $GITHUB_ENV - # - name: Cache dependencies - # id: dependency-cache - # uses: actions/cache@v2 - # with: - # path: | - # ~/.pyenv - # ~/Library/Caches/pip - # ~/Library/Caches/pipenv - # /usr/local/Cellar - # ~/github_brew_cache_entries.txt - # key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }} - # - name: Brew link restored dependencies - # if: steps.dependency-cache.outputs.cache-hit == 'true' - # run: | - # while read pkg; do - # brew link --force "$pkg" # `--force` for keg-only packages - # done < ~/github_brew_cache_entries.txt - # - name: Install dependencies - # run: ./tools/mac_setup.sh - # - name: Build openpilot - # run: eval "$(pyenv init -)" && scons -j$(nproc) - # - name: Remove pre-existing Homebrew packages for caching - # if: steps.dependency-cache.outputs.cache-hit != 'true' - # run: | - # cd /usr/local/Cellar - # new_cellar=$(ls -1) - # comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do - # if [[ $pkg != "zstd" ]]; then # caching step needs zstd - # rm -rf "$pkg" - # fi - # done - # comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt + build_mac: + name: build macos + runs-on: macos-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Determine pre-existing Homebrew packages + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + echo 'EXISTING_CELLAR<> $GITHUB_ENV + ls -1 /usr/local/Cellar >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Cache dependencies + id: dependency-cache + uses: actions/cache@v2 + with: + path: | + ~/.pyenv + ~/.local/share/virtualenvs/ + /usr/local/Cellar + ~/github_brew_cache_entries.txt + /tmp/scons_cache + key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }} + restore-keys: macos- + - name: Brew link restored dependencies + run: | + if [ -f ~/github_brew_cache_entries.txt ]; then + while read pkg; do + brew link --force "$pkg" # `--force` for keg-only packages + done < ~/github_brew_cache_entries.txt + else + echo "Cache entries not found" + fi + - name: Install dependencies + run: ./tools/mac_setup.sh + - name: Build openpilot + run: | + source tools/openpilot_env.sh + pipenv run selfdrive/manager/build.py + + # cleanup scons cache + rm -rf /tmp/scons_cache/ + pipenv run scons -j$(nproc) --cache-populate + - name: Remove pre-existing Homebrew packages for caching + if: steps.dependency-cache.outputs.cache-hit != 'true' + run: | + cd /usr/local/Cellar + new_cellar=$(ls -1) + comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do + if [[ $pkg != "zstd" ]]; then # caching step needs zstd + rm -rf "$pkg" + fi + done + comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt build_webcam: name: build webcam diff --git a/Pipfile b/Pipfile index 0ec00008b..bed4cdab7 100644 --- a/Pipfile +++ b/Pipfile @@ -38,7 +38,7 @@ tenacity = "*" [packages] atomicwrites = "*" -casadi = "*" +casadi = {version = "*", markers="platform_system != 'Darwin'"} cffi = "*" crcmod = "*" cryptography = "*" @@ -54,7 +54,7 @@ libusb1 = "*" nose = "*" numpy = "*" onnx = "*" -onnxruntime-gpu = "*" +onnxruntime-gpu = {version = "*", markers="platform_system != 'Darwin'"} pillow = "*" psutil = "*" pycapnp = "==1.1.0" diff --git a/Pipfile.lock b/Pipfile.lock index 15463e3e7..25cc81cbb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "658c6ab1fd200a6c84b1dec7a3b15b3d01ce911a76b1201bd4495b3031eac119" + "sha256": "40d9fa44e5b593786d40aa8a0276a1fbdfac46fbb78734687e6ef5b870813ed8" }, "pipfile-spec": 6, "requires": { @@ -81,6 +81,7 @@ "sha256:fbf39dcd63f1d3b63c300fce59b7ea678bd5ea1d014e1e090a5226600a4132cb" ], "index": "pypi", + "markers": "platform_system != 'Darwin'", "version": "==3.5.5" }, "certifi": { @@ -148,11 +149,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721", - "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c" + "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", + "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" ], "markers": "python_version >= '3'", - "version": "==2.0.9" + "version": "==2.0.10" }, "click": { "hashes": [ @@ -309,7 +310,7 @@ "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" ], - "markers": "python_full_version >= '3.6.1' and python_version < '4.0'", + "markers": "python_version < '4.0' and python_full_version >= '3.6.1'", "version": "==5.10.1" }, "itsdangerous": { @@ -558,6 +559,7 @@ "sha256:f6e056cd38265e1a158b72da234546291bb3cad663d81dcd66cbcbbce028e3a8" ], "index": "pypi", + "markers": "platform_system != 'Darwin'", "version": "==1.10.0" }, "pillow": { @@ -893,11 +895,11 @@ }, "requests": { "hashes": [ - "sha256:8e5643905bf20a308e25e4c1dd379117c09000bf8a82ebccc462cfb1b34a16b5", - "sha256:f71a09d7feba4a6b64ffd8e9d9bc60f9bf7d7e19fd0e04362acb1cfc2e3d98df" + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" ], "index": "pypi", - "version": "==2.27.0" + "version": "==2.27.1" }, "scons": { "hashes": [ @@ -942,6 +944,14 @@ "index": "pypi", "version": "==1.2.2" }, + "setuptools": { + "hashes": [ + "sha256:5c89b1a14a67ac5f0956f1cb0aeb7d1d3f4c8ba4e4e1ab7bf1af4933f9a2f0fe", + "sha256:675fcebecb43c32eb930481abf907619137547f4336206e4d673180242e1a278" + ], + "markers": "python_version >= '3.7'", + "version": "==60.2.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -979,7 +989,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tqdm": { @@ -995,7 +1005,7 @@ "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" ], - "markers": "python_version < '3.10' and python_version < '3.10'", + "markers": "python_version < '3.10'", "version": "==4.0.1" }, "urllib3": { @@ -1208,11 +1218,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721", - "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c" + "sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd", + "sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455" ], "markers": "python_version >= '3'", - "version": "==2.0.9" + "version": "==2.0.10" }, "control": { "hashes": [ @@ -1387,11 +1397,11 @@ }, "hypothesis": { "hashes": [ - "sha256:3cae408a92917fe288b0932eb3bb758f1585bf8b6e1d4ca4a666a2835c707db3", - "sha256:5002ce1f27fd94b53b0046ac7d20dab17fd74ebc6c90d401979588eadc46f84e" + "sha256:317f8d2f670fa69e258ab43e21c2befd413c559e386581f7e9641a80460b1063", + "sha256:803792d416ff71307d775fe760e2b2f07ca302a2c941b576629668092b9f3e3d" ], "index": "pypi", - "version": "==6.34.1" + "version": "==6.34.2" }, "identify": { "hashes": [ @@ -2032,11 +2042,11 @@ }, "requests": { "hashes": [ - "sha256:8e5643905bf20a308e25e4c1dd379117c09000bf8a82ebccc462cfb1b34a16b5", - "sha256:f71a09d7feba4a6b64ffd8e9d9bc60f9bf7d7e19fd0e04362acb1cfc2e3d98df" + "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61", + "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d" ], "index": "pypi", - "version": "==2.27.0" + "version": "==2.27.1" }, "reverse-geocoder": { "hashes": [ @@ -2080,6 +2090,14 @@ "index": "pypi", "version": "==1.7.3" }, + "setuptools": { + "hashes": [ + "sha256:5c89b1a14a67ac5f0956f1cb0aeb7d1d3f4c8ba4e4e1ab7bf1af4933f9a2f0fe", + "sha256:675fcebecb43c32eb930481abf907619137547f4336206e4d673180242e1a278" + ], + "markers": "python_version >= '3.7'", + "version": "==60.2.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -2195,7 +2213,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tomli": { @@ -2211,7 +2229,7 @@ "sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e", "sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b" ], - "markers": "python_version < '3.10' and python_version < '3.10'", + "markers": "python_version < '3.10'", "version": "==4.0.1" }, "urllib3": { diff --git a/SConstruct b/SConstruct index 6b45d60f2..832550b14 100644 --- a/SConstruct +++ b/SConstruct @@ -66,7 +66,7 @@ lenv = { "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath], "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath, - "ACADOS_SOURCE_DIR": Dir("#third_party/acados/acados").abspath, + "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath, "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath, "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer", } @@ -125,8 +125,10 @@ else: f"#third_party/libyuv/{yuv_dir}/lib", "/usr/local/lib", "/opt/homebrew/lib", + "/usr/local/Homebrew/Library", "/usr/local/opt/openssl/lib", "/opt/homebrew/opt/openssl/lib", + "/usr/local/Cellar", f"#third_party/acados/{arch}/lib", "/System/Library/Frameworks/OpenGL.framework/Libraries", ] @@ -134,6 +136,7 @@ else: cxxflags += ["-DGL_SILENCE_DEPRECATION"] cpppath += [ "/opt/homebrew/include", + "/usr/local/include", "/usr/local/opt/openssl/include", "/opt/homebrew/opt/openssl/include" ] @@ -235,6 +238,9 @@ env = Environment( tools=["default", "cython", "compilation_db"], ) +if arch == "Darwin": + env['RPATHPREFIX'] = "-rpath " + if GetOption('compile_db'): env.CompilationDatabase('compile_commands.json') @@ -299,6 +305,7 @@ if arch == "Darwin": qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules] qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")] qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"] + qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin")) elif arch == "aarch64": qt_env['QTDIR'] = "/system/comma/usr" qt_dirs = [ diff --git a/third_party/acados/Darwin/lib/libacados.dylib b/third_party/acados/Darwin/lib/libacados.dylib index 36dc230f7..056074e16 100755 Binary files a/third_party/acados/Darwin/lib/libacados.dylib and b/third_party/acados/Darwin/lib/libacados.dylib differ diff --git a/third_party/acados/Darwin/lib/libblasfeo.dylib b/third_party/acados/Darwin/lib/libblasfeo.dylib index cefef8894..e984307a2 100755 Binary files a/third_party/acados/Darwin/lib/libblasfeo.dylib and b/third_party/acados/Darwin/lib/libblasfeo.dylib differ diff --git a/third_party/acados/Darwin/lib/libhpipm.dylib b/third_party/acados/Darwin/lib/libhpipm.dylib index 5c0583861..a600c765b 100755 Binary files a/third_party/acados/Darwin/lib/libhpipm.dylib and b/third_party/acados/Darwin/lib/libhpipm.dylib differ diff --git a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib index 2ea5465be..185100d2c 100755 Binary files a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib and b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib differ diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index 71ffe4a92..fa1a23dd4 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/bin/bash + +set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ROOT="$(cd $DIR/../ && pwd)" @@ -7,15 +9,16 @@ ROOT="$(cd $DIR/../ && pwd)" if [[ $(command -v brew) == "" ]]; then echo "Installing Hombrew" /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" + echo "[ ] installed brew t=$SECONDS" fi +# TODO: remove protobuf,protobuf-c,swig when casadi can be pip installed brew bundle --file=- <<-EOS brew "cmake" +brew "cppcheck" brew "git-lfs" brew "zlib" brew "bzip2" -brew "rust" -brew "rustup-init" brew "capnp" brew "coreutils" brew "eigen" @@ -29,21 +32,20 @@ brew "openssl" brew "pyenv" brew "qt@5" brew "zeromq" +brew "protobuf" +brew "protobuf-c" +brew "swig" cask "gcc-arm-embedded" EOS +echo "[ ] finished brew install t=$SECONDS" + if [[ $SHELL == "/bin/zsh" ]]; then RC_FILE="$HOME/.zshrc" elif [[ $SHELL == "/bin/bash" ]]; then RC_FILE="$HOME/.bash_profile" fi -# TODO: get rid of this somehow -# Build requirements for macOS -# https://github.com/pyenv/pyenv/issues/1740 -# https://github.com/pyca/cryptography/blob/main/docs/installation.rst -rustup-init -y - export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib" export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib" export LDFLAGS="$LDFLAGS -L/usr/local/opt/openssl@1.1/lib" @@ -55,17 +57,40 @@ export PATH="$PATH:/usr/local/bin" # openpilot environment if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then - echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE - export PATH="$PATH:\"\$HOME/.cargo/bin\"" source "$ROOT/tools/openpilot_env.sh" echo "Added openpilot_env to RC file: $RC_FILE" fi # install python dependencies $ROOT/update_requirements.sh +eval "$(pyenv init --path)" +echo "[ ] installed python dependencies t=$SECONDS" + +# install casadi +VENV=`pipenv --venv` +PYTHON_VER=3.8 +PYTHON_VERSION=$(cat $ROOT/.python-version) +if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then + echo "-- casadi manual install" + cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz + tar -xzf casadi.tar.gz + cd casadi-ge6/ && mkdir -p build && cd build + cmake .. \ + -DWITH_PYTHON=ON \ + -DWITH_EXAMPLES=OFF \ + -DCMAKE_INSTALL_PREFIX:PATH=$VENV \ + -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \ + -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \ + -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \ + -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \ + -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0" + CFLAGS="-ferror-limit=0" make -j$(nproc) && make install +else + echo "---- casadi found in venv. skipping build ----" +fi echo -echo "---- FINISH OPENPILOT SETUP ----" -echo "Configure your active shell env by running:" +echo "---- OPENPILOT SETUP DONE ----" +echo "Open a new shell or configure your active shell env by running:" echo "source $RC_FILE" diff --git a/tools/openpilot_env.sh b/tools/openpilot_env.sh index ac73cc830..59108312a 100755 --- a/tools/openpilot_env.sh +++ b/tools/openpilot_env.sh @@ -9,13 +9,13 @@ if [ -z "$OPENPILOT_ENV" ]; then export PYENV_ROOT="$HOME/.pyenv" if [[ "$(uname)" == 'Linux' ]]; then - eval "$(pyenv init --path)" eval "$(pyenv virtualenv-init -)" elif [[ "$(uname)" == 'Darwin' ]]; then # msgq doesn't work on mac export ZMQ=1 export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES fi + eval "$(pyenv init --path)" eval "$(pyenv init -)" export OPENPILOT_ENV=1 diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh index dd29c995e..44ddb4f01 100755 --- a/tools/ubuntu_setup.sh +++ b/tools/ubuntu_setup.sh @@ -134,6 +134,6 @@ if [ -z "$OPENPILOT_ENV" ]; then fi echo -echo "---- FINISH OPENPILOT SETUP ----" -echo "Configure your active shell env by running:" +echo "---- OPENPILOT SETUP DONE ----" +echo "Open a new shell or configure your active shell env by running:" echo "source ~/.bashrc" diff --git a/update_requirements.sh b/update_requirements.sh index ebf99312b..ac9472dca 100755 --- a/update_requirements.sh +++ b/update_requirements.sh @@ -1,4 +1,6 @@ -#!/bin/bash -e +#!/bin/bash + +set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" cd $DIR @@ -13,15 +15,18 @@ export MAKEFLAGS="-j$(nproc)" PYENV_PYTHON_VERSION=$(cat .python-version) if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then - echo "pyenv update ..." - pyenv update + # no pyenv update on mac + if [ "$(uname)" == "Linux" ]; then + echo "pyenv update ..." + pyenv update + fi echo "python ${PYENV_PYTHON_VERSION} install ..." CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION} fi +eval "$(pyenv init --path)" -echo "pip install ..." +echo "update pip" pip install pip==21.3.1 -echo "pipenv install ..." pip install pipenv==2021.11.23 if [ -d "./xx" ]; then @@ -36,7 +41,7 @@ else RUN="" fi -echo "pip packages install ..." +echo "pip packages install..." pipenv install --dev --deploy --clear pyenv rehash @@ -45,4 +50,5 @@ if [ -f "$DIR/.pre-commit-config.yaml" ]; then $RUN pre-commit install [ -d "./xx" ] && (cd xx && $RUN pre-commit install) [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install) + echo "pre-commit hooks installed" fi