From 9529764549b8c59f7cf141ff3d2d9613ab53beab Mon Sep 17 00:00:00 2001 From: grekiki Date: Wed, 11 Nov 2020 21:14:51 +0100 Subject: [PATCH] Scons builder for cython extensions (#2485) * complie boardd without python * not good, but don't want to lose the file, because it works * clean a bit * update dbc * should build on CI * not good, but don't want to lose the file, because it works * clean a bit * should build on CI * remove unneeded path * reorder paths * reduce diff * and now it works?! * ... should work in CI * add kj, 30% chance to fix macos * pydebug * new way to find path * fix :) * tested * sanity check * repl. MacOS flags * hope it works * need more logs * need more logs2 * test if it works * should work on CI * correct python file * should not work * cleanup * real cleanup * more removals * 50% of file * transformations * fixed a hardcoded variable * more logs * simpl. * kalman * all donw if it passes tests * cleanup * reduce code by 20 lines if this works * fix bugs * cleanup * SharedLibrary * cleanup * ... * remove unused * CI fix maybe? * add more valid path * more logs * ...: * fix webcam CI * remove WError flag * deprecated is not an error * more Wno things * reduce diff, add Wno to env * don't import nonexistent stuff * SharedLibrary v2 * less custom env * renaming, remove SharedLibs * pack libs in envCython * experiment * better docker caching * whitespace * more docker caching * improvement * improvements Co-authored-by: Adeeb Shihadeh --- Dockerfile.openpilotci | 1 + SConstruct | 34 ++++++++++++++++++-- common/SConscript | 16 ++-------- common/clock.pyx | 1 + common/common_pyx_setup.py | 20 ------------ common/kalman/SConscript | 7 ++-- common/kalman/simple_kalman_impl.pyx | 1 + common/kalman/simple_kalman_setup.py | 10 ------ common/kalman/tests/test_simple_kalman.py | 3 ++ common/params_pyx.pyx | 2 +- common/params_pyx_setup.py | 33 ------------------- common/realtime.py | 2 +- common/transformations/SConscript | 9 ++---- common/transformations/setup.py | 20 ------------ common/transformations/transformations.pyx | 30 ++++++++++-------- release/files_common | 5 +-- selfdrive/boardd/SConscript | 6 ++-- selfdrive/boardd/boardd_setup.py | 22 ------------- site_scons/site_tools/cython.py | 37 ++++++++++++++++++++++ tools/webcam/Dockerfile | 1 + 20 files changed, 103 insertions(+), 157 deletions(-) delete mode 100644 common/common_pyx_setup.py delete mode 100644 common/kalman/simple_kalman_setup.py delete mode 100644 common/params_pyx_setup.py delete mode 100644 common/transformations/setup.py delete mode 100644 selfdrive/boardd/boardd_setup.py create mode 100644 site_scons/site_tools/cython.py diff --git a/Dockerfile.openpilotci b/Dockerfile.openpilotci index 0490d0a2d..ce60d3c26 100644 --- a/Dockerfile.openpilotci +++ b/Dockerfile.openpilotci @@ -12,6 +12,7 @@ COPY SConstruct \ COPY ./pyextra /tmp/openpilot/pyextra COPY ./phonelibs /tmp/openpilot/phonelibs +COPY ./site_scons /tmp/openpilot/site_scons COPY ./laika /tmp/openpilot/laika COPY ./laika_repo /tmp/openpilot/laika_repo COPY ./rednose /tmp/openpilot/rednose diff --git a/SConstruct b/SConstruct index b29f50aca..0846df94e 100644 --- a/SConstruct +++ b/SConstruct @@ -5,6 +5,8 @@ import shutil import subprocess import sys import platform +import numpy as np +from sysconfig import get_paths TICI = os.path.isfile('/TICI') Decider('MD5-timestamp') @@ -128,6 +130,10 @@ else: # change pythonpath to this lenv["PYTHONPATH"] = Dir("#").path +#Get the path for Python.h for cython linking +python_path = get_paths()['include'] +numpy_path = np.get_include() + env = Environment( ENV=lenv, CCFLAGS=[ @@ -159,6 +165,7 @@ env = Environment( "#phonelibs/linux/include", "#phonelibs/snpe/include", "#phonelibs/nanovg", + "#selfdrive/boardd", "#selfdrive/common", "#selfdrive/camerad", "#selfdrive/camerad/include", @@ -181,11 +188,14 @@ env = Environment( CXXFLAGS=["-std=c++1z"] + cxxflags, LIBPATH=libpath + [ "#cereal", + "#selfdrive/boardd", "#selfdrive/common", "#phonelibs", - ] + ], + CYTHONCFILESUFFIX=".cpp", + tools=["default", "cython"] ) - + if os.environ.get('SCONS_CACHE'): cache_dir = '/tmp/scons_cache' @@ -222,6 +232,25 @@ def abspath(x): # rpath works elsewhere return x[0].path.rsplit("/", 1)[1][:-3] +#Cython build enviroment +envCython = env.Clone() +envCython["CPPPATH"] += [python_path, numpy_path] +envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"] + +python_libs = [] +if arch == "Darwin": + envCython["LINKFLAGS"]=["-bundle", "-undefined", "dynamic_lookup"] +elif arch == "aarch64": + envCython["LINKFLAGS"]=["-shared"] + + python_libs.append(os.path.basename(python_path)) +else: + envCython["LINKFLAGS"]=["-pthread", "-shared"] + +envCython["LIBS"] = python_libs + +Export('envCython') + # still needed for apks zmq = 'zmq' Export('env', 'arch', 'real_arch', 'zmq', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY') @@ -277,6 +306,5 @@ SConscript(['selfdrive/ui/SConscript']) if arch != "Darwin": SConscript(['selfdrive/logcatd/SConscript']) - if arch == "x86_64": SConscript(['tools/lib/index_log/SConscript']) diff --git a/common/SConscript b/common/SConscript index d17b35fed..542e10d43 100644 --- a/common/SConscript +++ b/common/SConscript @@ -1,14 +1,4 @@ -Import('env', 'cython_dependencies') +Import('envCython') -# Build cython clock module -env.Command(['common_pyx.so', 'clock.cpp'], - cython_dependencies + ['common_pyx_setup.py', 'clock.pyx'], - "cd common && python3 common_pyx_setup.py build_ext --inplace") - -# Build cython params module -env.Command(['params_pyx.so', 'params_pyx.cpp'], - cython_dependencies + [ - 'params_pyx_setup.py', 'params_pyx.pyx', 'params_pxd.pxd', - '#selfdrive/common/params.cc', '#selfdrive/common/params.h', - '#selfdrive/common/util.c', '#selfdrive/common/util.h'], - "cd common && python3 params_pyx_setup.py build_ext --inplace") +envCython.Program('clock.so', 'clock.pyx') +envCython.Program('params_pyx.so', 'params_pyx.pyx') diff --git a/common/clock.pyx b/common/clock.pyx index 9702e488c..81333565c 100644 --- a/common/clock.pyx +++ b/common/clock.pyx @@ -1,3 +1,4 @@ +# distutils: language = c++ # cython: language_level = 3 from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t diff --git a/common/common_pyx_setup.py b/common/common_pyx_setup.py deleted file mode 100644 index d8f653e67..000000000 --- a/common/common_pyx_setup.py +++ /dev/null @@ -1,20 +0,0 @@ -from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module -from Cython.Build import cythonize - -from common.cython_hacks import BuildExtWithoutPlatformSuffix - -sourcefiles = ['clock.pyx'] -extra_compile_args = ["-std=c++1z"] - -setup(name='common', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "common_pyx", - language="c++", - sources=sourcefiles, - extra_compile_args=extra_compile_args, - ), - nthreads=4, - ), -) diff --git a/common/kalman/SConscript b/common/kalman/SConscript index 3d7011fe2..d60354c98 100644 --- a/common/kalman/SConscript +++ b/common/kalman/SConscript @@ -1,6 +1,3 @@ -Import('env', 'cython_dependencies') - -env.Command(['simple_kalman_impl.so'], - cython_dependencies + ['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'], - "cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace") +Import('envCython') +envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx') diff --git a/common/kalman/simple_kalman_impl.pyx b/common/kalman/simple_kalman_impl.pyx index e65efd577..16aefba2e 100644 --- a/common/kalman/simple_kalman_impl.pyx +++ b/common/kalman/simple_kalman_impl.pyx @@ -1,3 +1,4 @@ +# distutils: language = c++ # cython: language_level=3 cdef class KF1D: diff --git a/common/kalman/simple_kalman_setup.py b/common/kalman/simple_kalman_setup.py deleted file mode 100644 index d0781c9ff..000000000 --- a/common/kalman/simple_kalman_setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from distutils.core import Extension, setup - -from Cython.Build import cythonize - -from common.cython_hacks import BuildExtWithoutPlatformSuffix - -setup(name='Simple Kalman Implementation', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize(Extension("simple_kalman_impl", - ["simple_kalman_impl.pyx"]))) diff --git a/common/kalman/tests/test_simple_kalman.py b/common/kalman/tests/test_simple_kalman.py index 630875998..7b327918a 100644 --- a/common/kalman/tests/test_simple_kalman.py +++ b/common/kalman/tests/test_simple_kalman.py @@ -82,3 +82,6 @@ kf = KF1D(x0=[[x0_0], [x1_0]], kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000) kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000) self.assertTrue(kf_speed < kf_old_speed / 4) + +if __name__ == "__main__": + unittest.main() diff --git a/common/params_pyx.pyx b/common/params_pyx.pyx index b926cf4e9..b104011e2 100755 --- a/common/params_pyx.pyx +++ b/common/params_pyx.pyx @@ -2,7 +2,7 @@ # cython: language_level = 3 from libcpp cimport bool from libcpp.string cimport string -from params_pxd cimport Params as c_Params +from common.params_pxd cimport Params as c_Params import os import threading diff --git a/common/params_pyx_setup.py b/common/params_pyx_setup.py deleted file mode 100644 index 7fcdb6e7c..000000000 --- a/common/params_pyx_setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import subprocess -from distutils.core import Extension, setup -from Cython.Build import cythonize - -from common.cython_hacks import BuildExtWithoutPlatformSuffix -from common.basedir import BASEDIR -from common.hardware import TICI - -ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg - -sourcefiles = ['params_pyx.pyx'] -extra_compile_args = ["-std=c++1z"] - -if ARCH == "aarch64": - if TICI: - extra_compile_args += ["-DQCOM2"] - else: - extra_compile_args += ["-DQCOM"] - - -setup(name='common', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "params_pyx", - language="c++", - sources=sourcefiles, - include_dirs=[BASEDIR, os.path.join(BASEDIR, 'selfdrive')], - extra_compile_args=extra_compile_args - ) - ) -) diff --git a/common/realtime.py b/common/realtime.py index c883ec053..9f7315007 100644 --- a/common/realtime.py +++ b/common/realtime.py @@ -5,7 +5,7 @@ import time import multiprocessing from common.hardware import PC -from common.common_pyx import sec_since_boot # pylint: disable=no-name-in-module, import-error +from common.clock import sec_since_boot # pylint: disable=no-name-in-module, import-error # time step for each process diff --git a/common/transformations/SConscript b/common/transformations/SConscript index 0f7295225..adc9642a4 100644 --- a/common/transformations/SConscript +++ b/common/transformations/SConscript @@ -1,8 +1,3 @@ -Import('env', 'cython_dependencies') +Import('envCython') -d = Dir('.') - -env.Command(['transformations.so'], - cython_dependencies + ['transformations.pxd', 'transformations.pyx', - 'coordinates.cc', 'orientation.cc', 'coordinates.hpp', 'orientation.hpp'], - 'cd ' + d.path + ' && python3 setup.py build_ext --inplace') +envCython.Program('transformations.so', 'transformations.pyx') diff --git a/common/transformations/setup.py b/common/transformations/setup.py deleted file mode 100644 index 32546192d..000000000 --- a/common/transformations/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy - -from Cython.Build import cythonize -from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module -from common.cython_hacks import BuildExtWithoutPlatformSuffix - -setup( - name='Cython transformations wrapper', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "transformations", - sources=["transformations.pyx"], - language="c++", - extra_compile_args=["-std=c++1z", "-Wno-cpp"], - include_dirs=[numpy.get_include()], - ), - nthreads=4, - ) -) diff --git a/common/transformations/transformations.pyx b/common/transformations/transformations.pyx index 12d6e2ab5..ce80d90d2 100644 --- a/common/transformations/transformations.pyx +++ b/common/transformations/transformations.pyx @@ -1,18 +1,20 @@ -from transformations cimport Matrix3, Vector3, Quaternion -from transformations cimport ECEF, NED, Geodetic +# distutils: language = c++ +# cython: language_level = 3 +from common.transformations.transformations cimport Matrix3, Vector3, Quaternion +from common.transformations.transformations cimport ECEF, NED, Geodetic -from transformations cimport euler2quat as euler2quat_c -from transformations cimport quat2euler as quat2euler_c -from transformations cimport quat2rot as quat2rot_c -from transformations cimport rot2quat as rot2quat_c -from transformations cimport euler2rot as euler2rot_c -from transformations cimport rot2euler as rot2euler_c -from transformations cimport rot_matrix as rot_matrix_c -from transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c -from transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c -from transformations cimport geodetic2ecef as geodetic2ecef_c -from transformations cimport ecef2geodetic as ecef2geodetic_c -from transformations cimport LocalCoord_c +from common.transformations.transformations cimport euler2quat as euler2quat_c +from common.transformations.transformations cimport quat2euler as quat2euler_c +from common.transformations.transformations cimport quat2rot as quat2rot_c +from common.transformations.transformations cimport rot2quat as rot2quat_c +from common.transformations.transformations cimport euler2rot as euler2rot_c +from common.transformations.transformations cimport rot2euler as rot2euler_c +from common.transformations.transformations cimport rot_matrix as rot_matrix_c +from common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c +from common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c +from common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c +from common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c +from common.transformations.transformations cimport LocalCoord_c import cython diff --git a/release/files_common b/release/files_common index ec54ac1ac..f6b29d95c 100644 --- a/release/files_common +++ b/release/files_common @@ -12,6 +12,7 @@ CONTRIBUTING.md README.md RELEASES.md SAFETY.md +site_scons/site_tools/cython.py apk/ai.comma*.apk @@ -32,7 +33,6 @@ common/numpy_fast.py common/params.py common/params_pxd.pxd common/params_pyx.pyx -common/params_pyx_setup.py common/xattr.py common/profiler.py common/basedir.py @@ -43,7 +43,6 @@ common/text_window.py common/cython_hacks.py common/apk.py common/SConscript -common/common_pyx_setup.py common/kalman/.gitignore common/kalman/* @@ -53,7 +52,6 @@ common/transformations/camera.py common/transformations/model.py common/transformations/SConscript -common/transformations/setup.py common/transformations/coordinates.py common/transformations/coordinates.cc common/transformations/coordinates.hpp @@ -95,7 +93,6 @@ selfdrive/boardd/__init__.py selfdrive/boardd/boardd.cc selfdrive/boardd/boardd.py selfdrive/boardd/boardd_api_impl.pyx -selfdrive/boardd/boardd_setup.py selfdrive/boardd/can_list_to_can_capnp.cc selfdrive/boardd/panda.cc selfdrive/boardd/panda.h diff --git a/selfdrive/boardd/SConscript b/selfdrive/boardd/SConscript index 94e630a64..f2a1f3f7b 100644 --- a/selfdrive/boardd/SConscript +++ b/selfdrive/boardd/SConscript @@ -1,8 +1,6 @@ -Import('env', 'common', 'cereal', 'messaging', 'cython_dependencies') +Import('env', 'envCython', 'common', 'cereal', 'messaging') env.Program('boardd', ['boardd.cc', 'panda.cc', 'pigeon.cc'], LIBS=['usb-1.0', common, cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj']) env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc']) -env.Command(['boardd_api_impl.so', 'boardd_api_impl.cpp'], - cython_dependencies + ['libcan_list_to_can_capnp.a', 'boardd_api_impl.pyx', 'boardd_setup.py'], - "cd selfdrive/boardd && python3 boardd_setup.py build_ext --inplace") +envCython.Program('boardd_api_impl.so', 'boardd_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"]) diff --git a/selfdrive/boardd/boardd_setup.py b/selfdrive/boardd/boardd_setup.py deleted file mode 100644 index 01ca05248..000000000 --- a/selfdrive/boardd/boardd_setup.py +++ /dev/null @@ -1,22 +0,0 @@ -from distutils.core import Extension, setup -from Cython.Build import cythonize - -from common.cython_hacks import BuildExtWithoutPlatformSuffix - -libraries = ['can_list_to_can_capnp', 'capnp', 'kj'] - -setup(name='Boardd API Implementation', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "boardd_api_impl", - libraries=libraries, - library_dirs=[ - './', - ], - sources=['boardd_api_impl.pyx'], - language="c++", - extra_compile_args=["-std=c++1z", "-Wno-nullability-completeness"], - ) - ) -) diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py new file mode 100644 index 000000000..ad53f2831 --- /dev/null +++ b/site_scons/site_tools/cython.py @@ -0,0 +1,37 @@ +import SCons +from SCons.Action import Action + +cythonAction = Action("$CYTHONCOM") + +def create_builder(env): + try: + cython = env['BUILDERS']['Cython'] + except KeyError: + cython = SCons.Builder.Builder( + action = cythonAction, + emitter = {}, + suffix = cython_suffix_emitter, + single_source = 1) + env['BUILDERS']['Cython'] = cython + return cython + +def cython_suffix_emitter(env, source): + return "$CYTHONCFILESUFFIX" + +def generate(env): + env["CYTHON"] = "cythonize" + env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" + env["CYTHONCFILESUFFIX"] = ".cpp" + + c_file, _ = SCons.Tool.createCFileBuilders(env) + + c_file.suffix['.pyx'] = cython_suffix_emitter + c_file.add_action('.pyx', cythonAction) + + c_file.suffix['.py'] = cython_suffix_emitter + c_file.add_action('.py', cythonAction) + + create_builder(env) + +def exists(env): + return True diff --git a/tools/webcam/Dockerfile b/tools/webcam/Dockerfile index 53f5c8e86..3bcbffa0f 100644 --- a/tools/webcam/Dockerfile +++ b/tools/webcam/Dockerfile @@ -33,6 +33,7 @@ COPY SConstruct \ COPY ./pyextra /tmp/openpilot/pyextra COPY ./phonelibs /tmp/openpilot/phonelibs +COPY ./site_scons /tmp/openpilot/site_scons COPY ./laika /tmp/openpilot/laika COPY ./laika_repo /tmp/openpilot/laika_repo COPY ./rednose /tmp/openpilot/rednose