From 699dc63a881923661c1081b7b98ae600545d0f2f Mon Sep 17 00:00:00 2001 From: grekiki Date: Fri, 27 Nov 2020 01:11:23 +0100 Subject: [PATCH] Scons cython builder (#316) * migrate to home pc * style * this works * this is hacky * why doesn't this work * this should work * no lint * link against libdbc wtih abs path * new env Co-authored-by: Adeeb Shihadeh --- .gitignore | 1 + .pre-commit-config.yaml | 2 + Dockerfile | 1 + SConstruct | 45 +++++++++++----- can/SConscript | 10 ++-- can/common_pyx_setup.py | 94 --------------------------------- can/packer_pyx.pyx | 8 +-- can/parser_pyx.pyx | 4 +- site_scons/site_tools/cython.py | 38 +++++++++++++ 9 files changed, 85 insertions(+), 118 deletions(-) delete mode 100644 can/common_pyx_setup.py create mode 100644 site_scons/site_tools/cython.py diff --git a/.gitignore b/.gitignore index f259691..bf8570d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .mypy_cache/ *.pyc *.os +*.o *.tmp *.dylib .*.swp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf85d51..2f13d51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: rev: master hooks: - id: flake8 + exclude: 'site_scons/' args: - --ignore=E111,E114,E121,E124,E302,E501,E741 - --statistics @@ -24,6 +25,7 @@ repos: entry: pylint language: system types: [python] + exclude: 'site_scons/' args: - --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136 - repo: local diff --git a/Dockerfile b/Dockerfile index 4d976a5..9b1c555 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,7 @@ WORKDIR /project RUN git clone https://github.com/commaai/cereal.git /project/cereal COPY SConstruct . +COPY ./site_scons /project/site_scons COPY . /project/opendbc RUN rm -rf /project/opendbc/.git diff --git a/SConstruct b/SConstruct index 87615be..df9d0a9 100644 --- a/SConstruct +++ b/SConstruct @@ -1,24 +1,20 @@ -import Cython -import distutils import os import subprocess -import sys +import sysconfig zmq = 'zmq' arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() -# Rebuild cython extensions if python, distutils, or cython change -cython_dependencies = [Value(v) for v in (sys.version, distutils.__version__, Cython.__version__)] -Export('cython_dependencies') - cereal_dir = Dir('.') +python_path = sysconfig.get_paths()['include'] cpppath = [ - '#', - '#cereal', - "#cereal/messaging", - "#opendbc/can", - '/usr/lib/include', + '#', + '#cereal', + "#cereal/messaging", + "#opendbc/can", + '/usr/lib/include', + python_path ] AddOption('--test', @@ -45,10 +41,14 @@ env = Environment( ] + ccflags_asan, LDFLAGS=ldflags_asan, LINKFLAGS=ldflags_asan, - + LIBPATH=[ + "#opendbc/can/", + ], CFLAGS="-std=gnu11", CXXFLAGS="-std=c++1z", CPPPATH=cpppath, + CYTHONCFILESUFFIX=".cpp", + tools=["default", "cython"] ) Export('env', 'zmq', 'arch') @@ -57,5 +57,24 @@ cereal = [File('#cereal/libcereal.a')] messaging = [File('#cereal/libmessaging.a')] Export('cereal', 'messaging') + +envCython = env.Clone() +envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"] + +python_libs = [] +if arch == "Darwin": + envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] +elif arch == "aarch64": + envCython["LINKFLAGS"] = ["-shared"] + + python_libs.append(os.path.basename(python_path)) +else: + envCython["LINKFLAGS"] = ["-pthread", "-shared"] + +envCython["LIBS"] = python_libs + +Export('envCython') + + SConscript(['cereal/SConscript']) SConscript(['opendbc/can/SConscript']) diff --git a/can/SConscript b/can/SConscript index c8d4f34..4e67fc7 100644 --- a/can/SConscript +++ b/can/SConscript @@ -1,4 +1,4 @@ -Import('env', 'cereal', 'cython_dependencies') +Import('env', 'envCython', 'cereal') import os from opendbc.can.process_dbc import process @@ -13,10 +13,10 @@ for x in sorted(os.listdir('../')): dbc = env.Command(out_fn, in_fn, compile_dbc) dbcs.append(dbc) - libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"]) # Build packer and parser -env.Command(['packer_pyx.so', 'packer_pyx.cpp', 'parser_pyx.so', 'parser_pyx.cpp'], - cython_dependencies + [libdbc, cereal, 'common_pyx_setup.py', 'common.pxd', 'packer_pyx.pyx', 'parser_pyx.pyx', 'packer.cc', 'parser.cc'], - "cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace") +lenv = envCython.Clone() +lenv["LINKFLAGS"] += [libdbc[0].get_labspath()] +lenv.Program('parser_pyx.so', 'parser_pyx.pyx') +lenv.Program('packer_pyx.so', 'packer_pyx.pyx') diff --git a/can/common_pyx_setup.py b/can/common_pyx_setup.py deleted file mode 100644 index a729984..0000000 --- a/can/common_pyx_setup.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import subprocess -import sysconfig -import platform -from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module - -from Cython.Build import cythonize -from Cython.Distutils import build_ext - - -ANNOTATE = os.getenv('ANNOTATE') is not None -BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../")) - - -def get_ext_filename_without_platform_suffix(filename): - name, ext = os.path.splitext(filename) - ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') - - if ext_suffix == ext: - return filename - - ext_suffix = ext_suffix.replace(ext, '') - idx = name.find(ext_suffix) - - if idx == -1: - return filename - else: - return name[:idx] + ext - - -class BuildExtWithoutPlatformSuffix(build_ext): - def get_ext_filename(self, ext_name): - filename = super().get_ext_filename(ext_name) - return get_ext_filename_without_platform_suffix(filename) - - -extra_compile_args = ["-std=c++1z", "-Wno-nullability-completeness"] -ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() # pylint: disable=unexpected-keyword-arg -if ARCH == "aarch64": - extra_compile_args += ["-Wno-deprecated-register"] - -if platform.system() == "Darwin": - libdbc = "libdbc.dylib" -else: - libdbc = "libdbc.so" - -extra_link_args = [os.path.join(BASEDIR, 'opendbc', 'can', libdbc)] -include_dirs = [ - BASEDIR, - os.path.join(BASEDIR, 'phonelibs'), -] - -# Build CAN Parser - -setup(name='CAN parser', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "parser_pyx", - language="c++", - sources=['parser_pyx.pyx'], - extra_compile_args=extra_compile_args, - include_dirs=include_dirs, - extra_link_args=extra_link_args, - ), - nthreads=4, - annotate=ANNOTATE - ), -) - -if platform.system() == "Darwin": - os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib parser_pyx.so") - - -# Build CAN Packer - -setup(name='CAN packer', - cmdclass={'build_ext': BuildExtWithoutPlatformSuffix}, - ext_modules=cythonize( - Extension( - "packer_pyx", - language="c++", - sources=['packer_pyx.pyx'], - extra_compile_args=extra_compile_args, - include_dirs=include_dirs, - extra_link_args=extra_link_args, - ), - nthreads=4, - annotate=ANNOTATE - ), -) - -if platform.system() == "Darwin": - os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib packer_pyx.so") diff --git a/can/packer_pyx.pyx b/can/packer_pyx.pyx index 14e2f81..22b4b56 100644 --- a/can/packer_pyx.pyx +++ b/can/packer_pyx.pyx @@ -8,8 +8,8 @@ from libcpp.string cimport string from libcpp cimport bool from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY -from common cimport CANPacker as cpp_CANPacker -from common cimport dbc_lookup, SignalPackValue, DBC +from .common cimport CANPacker as cpp_CANPacker +from .common cimport dbc_lookup, SignalPackValue, DBC cdef class CANPacker: @@ -22,8 +22,8 @@ cdef class CANPacker: def __init__(self, dbc_name): self.dbc = dbc_lookup(dbc_name) if not self.dbc: - raise RuntimeError("Can't lookup" + dbc_name) - + raise RuntimeError(f"Can't lookup {dbc_name}") + self.packer = new cpp_CANPacker(dbc_name) num_msgs = self.dbc[0].num_msgs for i in range(num_msgs): diff --git a/can/parser_pyx.pyx b/can/parser_pyx.pyx index 913f582..39f00a5 100644 --- a/can/parser_pyx.pyx +++ b/can/parser_pyx.pyx @@ -8,8 +8,8 @@ from libc.stdint cimport uint32_t, uint64_t, uint16_t from libcpp.map cimport map from libcpp cimport bool -from common cimport CANParser as cpp_CANParser -from common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC +from .common cimport CANParser as cpp_CANParser +from .common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC import os import numbers diff --git a/site_scons/site_tools/cython.py b/site_scons/site_tools/cython.py new file mode 100644 index 0000000..45ba797 --- /dev/null +++ b/site_scons/site_tools/cython.py @@ -0,0 +1,38 @@ +import SCons +from SCons.Action import Action + +cythonAction = Action("$CYTHONCOM") + +def create_builder(env): + try: + cython = env['BUILDERS']['Cython'] + except KeyError: + cython = SCons.Builder.Builder( + action = cythonAction, + emitter = {}, + suffix = cython_suffix_emitter, + single_source = 1 + ) + env['BUILDERS']['Cython'] = cython + return cython + +def cython_suffix_emitter(env, source): + return "$CYTHONCFILESUFFIX" + +def generate(env): + env["CYTHON"] = "cythonize" + env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE" + env["CYTHONCFILESUFFIX"] = ".cpp" + + c_file, _ = SCons.Tool.createCFileBuilders(env) + + c_file.suffix['.pyx'] = cython_suffix_emitter + c_file.add_action('.pyx', cythonAction) + + c_file.suffix['.py'] = cython_suffix_emitter + c_file.add_action('.py', cythonAction) + + create_builder(env) + +def exists(env): + return True