Scons builder for cython extensions (#2485)
* complie boardd without python * not good, but don't want to lose the file, because it works * clean a bit * update dbc * should build on CI * not good, but don't want to lose the file, because it works * clean a bit * should build on CI * remove unneeded path * reorder paths * reduce diff * and now it works?! * ... should work in CI * add kj, 30% chance to fix macos * pydebug * new way to find path * fix :) * tested * sanity check * repl. MacOS flags * hope it works * need more logs * need more logs2 * test if it works * should work on CI * correct python file * should not work * cleanup * real cleanup * more removals * 50% of file * transformations * fixed a hardcoded variable * more logs * simpl. * kalman * all donw if it passes tests * cleanup * reduce code by 20 lines if this works * fix bugs * cleanup * SharedLibrary * cleanup * ... * remove unused * CI fix maybe? * add more valid path * more logs * ...: * fix webcam CI * remove WError flag * deprecated is not an error * more Wno things * reduce diff, add Wno to env * don't import nonexistent stuff * SharedLibrary v2 * less custom env * renaming, remove SharedLibs * pack libs in envCython * experiment * better docker caching * whitespace * more docker caching * improvement * improvements Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>pull/2527/head
grekiki
GitHub
parent
8896b7a791
commit
9529764549
|
|
@ -12,6 +12,7 @@ COPY SConstruct \
|
|||
|
||||
COPY ./pyextra /tmp/openpilot/pyextra
|
||||
COPY ./phonelibs /tmp/openpilot/phonelibs
|
||||
COPY ./site_scons /tmp/openpilot/site_scons
|
||||
COPY ./laika /tmp/openpilot/laika
|
||||
COPY ./laika_repo /tmp/openpilot/laika_repo
|
||||
COPY ./rednose /tmp/openpilot/rednose
|
||||
|
|
|
|||
34
SConstruct
34
SConstruct
|
|
@ -5,6 +5,8 @@ import shutil
|
|||
import subprocess
|
||||
import sys
|
||||
import platform
|
||||
import numpy as np
|
||||
from sysconfig import get_paths
|
||||
|
||||
TICI = os.path.isfile('/TICI')
|
||||
Decider('MD5-timestamp')
|
||||
|
|
@ -128,6 +130,10 @@ else:
|
|||
# change pythonpath to this
|
||||
lenv["PYTHONPATH"] = Dir("#").path
|
||||
|
||||
#Get the path for Python.h for cython linking
|
||||
python_path = get_paths()['include']
|
||||
numpy_path = np.get_include()
|
||||
|
||||
env = Environment(
|
||||
ENV=lenv,
|
||||
CCFLAGS=[
|
||||
|
|
@ -159,6 +165,7 @@ env = Environment(
|
|||
"#phonelibs/linux/include",
|
||||
"#phonelibs/snpe/include",
|
||||
"#phonelibs/nanovg",
|
||||
"#selfdrive/boardd",
|
||||
"#selfdrive/common",
|
||||
"#selfdrive/camerad",
|
||||
"#selfdrive/camerad/include",
|
||||
|
|
@ -181,11 +188,14 @@ env = Environment(
|
|||
CXXFLAGS=["-std=c++1z"] + cxxflags,
|
||||
LIBPATH=libpath + [
|
||||
"#cereal",
|
||||
"#selfdrive/boardd",
|
||||
"#selfdrive/common",
|
||||
"#phonelibs",
|
||||
]
|
||||
],
|
||||
CYTHONCFILESUFFIX=".cpp",
|
||||
tools=["default", "cython"]
|
||||
)
|
||||
|
||||
|
||||
if os.environ.get('SCONS_CACHE'):
|
||||
cache_dir = '/tmp/scons_cache'
|
||||
|
||||
|
|
@ -222,6 +232,25 @@ def abspath(x):
|
|||
# rpath works elsewhere
|
||||
return x[0].path.rsplit("/", 1)[1][:-3]
|
||||
|
||||
#Cython build enviroment
|
||||
envCython = env.Clone()
|
||||
envCython["CPPPATH"] += [python_path, numpy_path]
|
||||
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
|
||||
|
||||
python_libs = []
|
||||
if arch == "Darwin":
|
||||
envCython["LINKFLAGS"]=["-bundle", "-undefined", "dynamic_lookup"]
|
||||
elif arch == "aarch64":
|
||||
envCython["LINKFLAGS"]=["-shared"]
|
||||
|
||||
python_libs.append(os.path.basename(python_path))
|
||||
else:
|
||||
envCython["LINKFLAGS"]=["-pthread", "-shared"]
|
||||
|
||||
envCython["LIBS"] = python_libs
|
||||
|
||||
Export('envCython')
|
||||
|
||||
# still needed for apks
|
||||
zmq = 'zmq'
|
||||
Export('env', 'arch', 'real_arch', 'zmq', 'SHARED', 'USE_WEBCAM', 'QCOM_REPLAY')
|
||||
|
|
@ -277,6 +306,5 @@ SConscript(['selfdrive/ui/SConscript'])
|
|||
if arch != "Darwin":
|
||||
SConscript(['selfdrive/logcatd/SConscript'])
|
||||
|
||||
|
||||
if arch == "x86_64":
|
||||
SConscript(['tools/lib/index_log/SConscript'])
|
||||
|
|
|
|||
|
|
@ -1,14 +1,4 @@
|
|||
Import('env', 'cython_dependencies')
|
||||
Import('envCython')
|
||||
|
||||
# Build cython clock module
|
||||
env.Command(['common_pyx.so', 'clock.cpp'],
|
||||
cython_dependencies + ['common_pyx_setup.py', 'clock.pyx'],
|
||||
"cd common && python3 common_pyx_setup.py build_ext --inplace")
|
||||
|
||||
# Build cython params module
|
||||
env.Command(['params_pyx.so', 'params_pyx.cpp'],
|
||||
cython_dependencies + [
|
||||
'params_pyx_setup.py', 'params_pyx.pyx', 'params_pxd.pxd',
|
||||
'#selfdrive/common/params.cc', '#selfdrive/common/params.h',
|
||||
'#selfdrive/common/util.c', '#selfdrive/common/util.h'],
|
||||
"cd common && python3 params_pyx_setup.py build_ext --inplace")
|
||||
envCython.Program('clock.so', 'clock.pyx')
|
||||
envCython.Program('params_pyx.so', 'params_pyx.pyx')
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# distutils: language = c++
|
||||
# cython: language_level = 3
|
||||
from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
|
||||
from Cython.Build import cythonize
|
||||
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
|
||||
sourcefiles = ['clock.pyx']
|
||||
extra_compile_args = ["-std=c++1z"]
|
||||
|
||||
setup(name='common',
|
||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
||||
ext_modules=cythonize(
|
||||
Extension(
|
||||
"common_pyx",
|
||||
language="c++",
|
||||
sources=sourcefiles,
|
||||
extra_compile_args=extra_compile_args,
|
||||
),
|
||||
nthreads=4,
|
||||
),
|
||||
)
|
||||
|
|
@ -1,6 +1,3 @@
|
|||
Import('env', 'cython_dependencies')
|
||||
|
||||
env.Command(['simple_kalman_impl.so'],
|
||||
cython_dependencies + ['simple_kalman_impl.pyx', 'simple_kalman_impl.pxd', 'simple_kalman_setup.py'],
|
||||
"cd common/kalman && python3 simple_kalman_setup.py build_ext --inplace")
|
||||
Import('envCython')
|
||||
|
||||
envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx')
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# distutils: language = c++
|
||||
# cython: language_level=3
|
||||
|
||||
cdef class KF1D:
|
||||
|
|
|
|||
|
|
@ -1,10 +0,0 @@
|
|||
from distutils.core import Extension, setup
|
||||
|
||||
from Cython.Build import cythonize
|
||||
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
|
||||
setup(name='Simple Kalman Implementation',
|
||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
||||
ext_modules=cythonize(Extension("simple_kalman_impl",
|
||||
["simple_kalman_impl.pyx"])))
|
||||
|
|
@ -82,3 +82,6 @@ kf = KF1D(x0=[[x0_0], [x1_0]],
|
|||
kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
|
||||
kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
|
||||
self.assertTrue(kf_speed < kf_old_speed / 4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# cython: language_level = 3
|
||||
from libcpp cimport bool
|
||||
from libcpp.string cimport string
|
||||
from params_pxd cimport Params as c_Params
|
||||
from common.params_pxd cimport Params as c_Params
|
||||
|
||||
import os
|
||||
import threading
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
import os
|
||||
import subprocess
|
||||
from distutils.core import Extension, setup
|
||||
from Cython.Build import cythonize
|
||||
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
from common.basedir import BASEDIR
|
||||
from common.hardware import TICI
|
||||
|
||||
ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg
|
||||
|
||||
sourcefiles = ['params_pyx.pyx']
|
||||
extra_compile_args = ["-std=c++1z"]
|
||||
|
||||
if ARCH == "aarch64":
|
||||
if TICI:
|
||||
extra_compile_args += ["-DQCOM2"]
|
||||
else:
|
||||
extra_compile_args += ["-DQCOM"]
|
||||
|
||||
|
||||
setup(name='common',
|
||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
||||
ext_modules=cythonize(
|
||||
Extension(
|
||||
"params_pyx",
|
||||
language="c++",
|
||||
sources=sourcefiles,
|
||||
include_dirs=[BASEDIR, os.path.join(BASEDIR, 'selfdrive')],
|
||||
extra_compile_args=extra_compile_args
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -5,7 +5,7 @@ import time
|
|||
import multiprocessing
|
||||
|
||||
from common.hardware import PC
|
||||
from common.common_pyx import sec_since_boot # pylint: disable=no-name-in-module, import-error
|
||||
from common.clock import sec_since_boot # pylint: disable=no-name-in-module, import-error
|
||||
|
||||
|
||||
# time step for each process
|
||||
|
|
|
|||
|
|
@ -1,8 +1,3 @@
|
|||
Import('env', 'cython_dependencies')
|
||||
Import('envCython')
|
||||
|
||||
d = Dir('.')
|
||||
|
||||
env.Command(['transformations.so'],
|
||||
cython_dependencies + ['transformations.pxd', 'transformations.pyx',
|
||||
'coordinates.cc', 'orientation.cc', 'coordinates.hpp', 'orientation.hpp'],
|
||||
'cd ' + d.path + ' && python3 setup.py build_ext --inplace')
|
||||
envCython.Program('transformations.so', 'transformations.pyx')
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
import numpy
|
||||
|
||||
from Cython.Build import cythonize
|
||||
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
|
||||
setup(
|
||||
name='Cython transformations wrapper',
|
||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
||||
ext_modules=cythonize(
|
||||
Extension(
|
||||
"transformations",
|
||||
sources=["transformations.pyx"],
|
||||
language="c++",
|
||||
extra_compile_args=["-std=c++1z", "-Wno-cpp"],
|
||||
include_dirs=[numpy.get_include()],
|
||||
),
|
||||
nthreads=4,
|
||||
)
|
||||
)
|
||||
|
|
@ -1,18 +1,20 @@
|
|||
from transformations cimport Matrix3, Vector3, Quaternion
|
||||
from transformations cimport ECEF, NED, Geodetic
|
||||
# distutils: language = c++
|
||||
# cython: language_level = 3
|
||||
from common.transformations.transformations cimport Matrix3, Vector3, Quaternion
|
||||
from common.transformations.transformations cimport ECEF, NED, Geodetic
|
||||
|
||||
from transformations cimport euler2quat as euler2quat_c
|
||||
from transformations cimport quat2euler as quat2euler_c
|
||||
from transformations cimport quat2rot as quat2rot_c
|
||||
from transformations cimport rot2quat as rot2quat_c
|
||||
from transformations cimport euler2rot as euler2rot_c
|
||||
from transformations cimport rot2euler as rot2euler_c
|
||||
from transformations cimport rot_matrix as rot_matrix_c
|
||||
from transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
|
||||
from transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
|
||||
from transformations cimport geodetic2ecef as geodetic2ecef_c
|
||||
from transformations cimport ecef2geodetic as ecef2geodetic_c
|
||||
from transformations cimport LocalCoord_c
|
||||
from common.transformations.transformations cimport euler2quat as euler2quat_c
|
||||
from common.transformations.transformations cimport quat2euler as quat2euler_c
|
||||
from common.transformations.transformations cimport quat2rot as quat2rot_c
|
||||
from common.transformations.transformations cimport rot2quat as rot2quat_c
|
||||
from common.transformations.transformations cimport euler2rot as euler2rot_c
|
||||
from common.transformations.transformations cimport rot2euler as rot2euler_c
|
||||
from common.transformations.transformations cimport rot_matrix as rot_matrix_c
|
||||
from common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
|
||||
from common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
|
||||
from common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
|
||||
from common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
|
||||
from common.transformations.transformations cimport LocalCoord_c
|
||||
|
||||
|
||||
import cython
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ CONTRIBUTING.md
|
|||
README.md
|
||||
RELEASES.md
|
||||
SAFETY.md
|
||||
site_scons/site_tools/cython.py
|
||||
|
||||
apk/ai.comma*.apk
|
||||
|
||||
|
|
@ -32,7 +33,6 @@ common/numpy_fast.py
|
|||
common/params.py
|
||||
common/params_pxd.pxd
|
||||
common/params_pyx.pyx
|
||||
common/params_pyx_setup.py
|
||||
common/xattr.py
|
||||
common/profiler.py
|
||||
common/basedir.py
|
||||
|
|
@ -43,7 +43,6 @@ common/text_window.py
|
|||
common/cython_hacks.py
|
||||
common/apk.py
|
||||
common/SConscript
|
||||
common/common_pyx_setup.py
|
||||
|
||||
common/kalman/.gitignore
|
||||
common/kalman/*
|
||||
|
|
@ -53,7 +52,6 @@ common/transformations/camera.py
|
|||
common/transformations/model.py
|
||||
|
||||
common/transformations/SConscript
|
||||
common/transformations/setup.py
|
||||
common/transformations/coordinates.py
|
||||
common/transformations/coordinates.cc
|
||||
common/transformations/coordinates.hpp
|
||||
|
|
@ -95,7 +93,6 @@ selfdrive/boardd/__init__.py
|
|||
selfdrive/boardd/boardd.cc
|
||||
selfdrive/boardd/boardd.py
|
||||
selfdrive/boardd/boardd_api_impl.pyx
|
||||
selfdrive/boardd/boardd_setup.py
|
||||
selfdrive/boardd/can_list_to_can_capnp.cc
|
||||
selfdrive/boardd/panda.cc
|
||||
selfdrive/boardd/panda.h
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
Import('env', 'common', 'cereal', 'messaging', 'cython_dependencies')
|
||||
Import('env', 'envCython', 'common', 'cereal', 'messaging')
|
||||
|
||||
env.Program('boardd', ['boardd.cc', 'panda.cc', 'pigeon.cc'], LIBS=['usb-1.0', common, cereal, messaging, 'pthread', 'zmq', 'capnp', 'kj'])
|
||||
env.Library('libcan_list_to_can_capnp', ['can_list_to_can_capnp.cc'])
|
||||
|
||||
env.Command(['boardd_api_impl.so', 'boardd_api_impl.cpp'],
|
||||
cython_dependencies + ['libcan_list_to_can_capnp.a', 'boardd_api_impl.pyx', 'boardd_setup.py'],
|
||||
"cd selfdrive/boardd && python3 boardd_setup.py build_ext --inplace")
|
||||
envCython.Program('boardd_api_impl.so', 'boardd_api_impl.pyx', LIBS=["can_list_to_can_capnp", 'capnp', 'kj'] + envCython["LIBS"])
|
||||
|
|
|
|||
|
|
@ -1,22 +0,0 @@
|
|||
from distutils.core import Extension, setup
|
||||
from Cython.Build import cythonize
|
||||
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
|
||||
libraries = ['can_list_to_can_capnp', 'capnp', 'kj']
|
||||
|
||||
setup(name='Boardd API Implementation',
|
||||
cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
|
||||
ext_modules=cythonize(
|
||||
Extension(
|
||||
"boardd_api_impl",
|
||||
libraries=libraries,
|
||||
library_dirs=[
|
||||
'./',
|
||||
],
|
||||
sources=['boardd_api_impl.pyx'],
|
||||
language="c++",
|
||||
extra_compile_args=["-std=c++1z", "-Wno-nullability-completeness"],
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import SCons
|
||||
from SCons.Action import Action
|
||||
|
||||
cythonAction = Action("$CYTHONCOM")
|
||||
|
||||
def create_builder(env):
|
||||
try:
|
||||
cython = env['BUILDERS']['Cython']
|
||||
except KeyError:
|
||||
cython = SCons.Builder.Builder(
|
||||
action = cythonAction,
|
||||
emitter = {},
|
||||
suffix = cython_suffix_emitter,
|
||||
single_source = 1)
|
||||
env['BUILDERS']['Cython'] = cython
|
||||
return cython
|
||||
|
||||
def cython_suffix_emitter(env, source):
|
||||
return "$CYTHONCFILESUFFIX"
|
||||
|
||||
def generate(env):
|
||||
env["CYTHON"] = "cythonize"
|
||||
env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
|
||||
env["CYTHONCFILESUFFIX"] = ".cpp"
|
||||
|
||||
c_file, _ = SCons.Tool.createCFileBuilders(env)
|
||||
|
||||
c_file.suffix['.pyx'] = cython_suffix_emitter
|
||||
c_file.add_action('.pyx', cythonAction)
|
||||
|
||||
c_file.suffix['.py'] = cython_suffix_emitter
|
||||
c_file.add_action('.py', cythonAction)
|
||||
|
||||
create_builder(env)
|
||||
|
||||
def exists(env):
|
||||
return True
|
||||
|
|
@ -33,6 +33,7 @@ COPY SConstruct \
|
|||
|
||||
COPY ./pyextra /tmp/openpilot/pyextra
|
||||
COPY ./phonelibs /tmp/openpilot/phonelibs
|
||||
COPY ./site_scons /tmp/openpilot/site_scons
|
||||
COPY ./laika /tmp/openpilot/laika
|
||||
COPY ./laika_repo /tmp/openpilot/laika_repo
|
||||
COPY ./rednose /tmp/openpilot/rednose
|
||||
|
|
|
|||
Loading…
Reference in New Issue