From 8ba963cfa3da8e712e94c2429cfd6f064b1b4c69 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 16 Oct 2019 15:12:39 +1100 Subject: [PATCH] tools/makemanifest.py: Eval relative paths w.r.t. current manifest file. When loading a manifest file, e.g. by include(), it will chdir first to the directory of that manifest. This means that all file operations within a manifest are relative to that manifest's location. As a consequence of this, additional environment variables are needed to find absolute paths, so the following are added: $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR). And rename $(MPY) to $(MPY_DIR) to be consistent. Existing manifests are updated to match. --- ports/esp32/boards/manifest.py | 12 ++-- ports/esp32/boards/manifest_release.py | 12 ++-- ports/esp8266/boards/manifest.py | 8 +-- ports/stm32/boards/manifest.py | 6 +- ports/unix/manifest.py | 4 +- py/mkenv.mk | 2 + py/mkrules.mk | 2 +- tools/makemanifest.py | 83 +++++++++++++++++++------- 8 files changed, 85 insertions(+), 44 deletions(-) diff --git a/ports/esp32/boards/manifest.py b/ports/esp32/boards/manifest.py index 3da8af57f..2b07639ee 100644 --- a/ports/esp32/boards/manifest.py +++ b/ports/esp32/boards/manifest.py @@ -1,6 +1,6 @@ -freeze('modules') -freeze('$(MPY)/tools', ('upip.py', 'upip_utarfile.py')) -freeze('$(MPY)/ports/esp8266/modules', 'ntptime.py') -freeze('$(MPY)/ports/esp8266/modules', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',)) -freeze('$(MPY)/drivers/dht', 'dht.py') -freeze('$(MPY)/drivers/onewire') +freeze('$(PORT_DIR)/modules') +freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py')) +freeze('$(MPY_DIR)/ports/esp8266/modules', 'ntptime.py') +freeze('$(MPY_DIR)/ports/esp8266/modules', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',)) +freeze('$(MPY_DIR)/drivers/dht', 'dht.py') +freeze('$(MPY_DIR)/drivers/onewire') diff --git a/ports/esp32/boards/manifest_release.py b/ports/esp32/boards/manifest_release.py index e56704d02..9c898af26 100644 --- a/ports/esp32/boards/manifest_release.py +++ b/ports/esp32/boards/manifest_release.py @@ -1,8 +1,6 @@ -include('boards/manifest.py') +include('manifest.py') -LIB = '../../../micropython-lib' - -freeze(LIB + '/upysh', 'upysh.py') -freeze(LIB + '/urequests', 'urequests.py') -freeze(LIB + '/umqtt.simple', 'umqtt/simple.py') -freeze(LIB + '/umqtt.robust', 'umqtt/robust.py') +freeze('$(MPY_LIB_DIR)/upysh', 'upysh.py') +freeze('$(MPY_LIB_DIR)/urequests', 'urequests.py') +freeze('$(MPY_LIB_DIR)/umqtt.simple', 'umqtt/simple.py') +freeze('$(MPY_LIB_DIR)/umqtt.robust', 'umqtt/robust.py') diff --git a/ports/esp8266/boards/manifest.py b/ports/esp8266/boards/manifest.py index 1264a2268..779e84088 100644 --- a/ports/esp8266/boards/manifest.py +++ b/ports/esp8266/boards/manifest.py @@ -1,4 +1,4 @@ -freeze('modules') -freeze('$(MPY)/tools', ('upip.py', 'upip_utarfile.py')) -freeze('$(MPY)/drivers/dht', 'dht.py') -freeze('$(MPY)/drivers/onewire') +freeze('$(PORT_DIR)/modules') +freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py')) +freeze('$(MPY_DIR)/drivers/dht', 'dht.py') +freeze('$(MPY_DIR)/drivers/onewire') diff --git a/ports/stm32/boards/manifest.py b/ports/stm32/boards/manifest.py index 99b08cca0..41b728fa2 100644 --- a/ports/stm32/boards/manifest.py +++ b/ports/stm32/boards/manifest.py @@ -1,3 +1,3 @@ -freeze('$(MPY)/drivers/dht', 'dht.py') -freeze('$(MPY)/drivers/display', ('lcd160cr.py', 'lcd160cr_test.py')) -freeze('$(MPY)/drivers/onewire', 'onewire.py') +freeze('$(MPY_DIR)/drivers/dht', 'dht.py') +freeze('$(MPY_DIR)/drivers/display', ('lcd160cr.py', 'lcd160cr_test.py')) +freeze('$(MPY_DIR)/drivers/onewire', 'onewire.py') diff --git a/ports/unix/manifest.py b/ports/unix/manifest.py index 3f332446d..666b4c0ab 100644 --- a/ports/unix/manifest.py +++ b/ports/unix/manifest.py @@ -1,2 +1,2 @@ -freeze_as_mpy('$(MPY)/tools', 'upip.py') -freeze_as_mpy('$(MPY)/tools', 'upip_utarfile.py', opt=3) +freeze_as_mpy('$(MPY_DIR)/tools', 'upip.py') +freeze_as_mpy('$(MPY_DIR)/tools', 'upip_utarfile.py', opt=3) diff --git a/py/mkenv.mk b/py/mkenv.mk index 70d937c45..3efeb1816 100644 --- a/py/mkenv.mk +++ b/py/mkenv.mk @@ -66,6 +66,8 @@ MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py MPY_CROSS = $(TOP)/mpy-cross/mpy-cross MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py +MPY_LIB_DIR = $(TOP)/../micropython-lib + all: .PHONY: all diff --git a/py/mkrules.mk b/py/mkrules.mk index 2a7e1980c..b43c6b8e3 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -100,7 +100,7 @@ $(HEADER_BUILD): ifneq ($(FROZEN_MANIFEST),) # to build frozen_content.c from a manifest $(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h - $(Q)$(MAKE_MANIFEST) -o $@ $(TOP) $(BUILD) "$(MPY_CROSS_FLAGS)" $(FROZEN_MANIFEST) + $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) $(FROZEN_MANIFEST) endif ifneq ($(FROZEN_DIR),) diff --git a/tools/makemanifest.py b/tools/makemanifest.py index 3017d7a21..9889d5075 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -38,14 +38,22 @@ def include(manifest): The manifest argument can be a string (filename) or an iterable of strings. + + Relative paths are resolved with respect to the current manifest file. """ if not isinstance(manifest, str): for m in manifest: include(m) else: + manifest = convert_path(manifest) with open(manifest) as f: + # Make paths relative to this manifest file while processing it. + # Applies to includes and input files. + prev_cwd = os.getcwd() + os.chdir(os.path.dirname(manifest)) exec(f.read()) + os.chdir(prev_cwd) def freeze(path, script=None, opt=0): """Freeze the input, automatically determining its type. A .py script @@ -57,6 +65,10 @@ def freeze(path, script=None, opt=0): the module will start after `path`, ie `path` is excluded from the module name. + If `path` is relative, it is resolved to the current manifest.py. + Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need + to access specific paths. + If `script` is None all files in `path` will be frozen. If `script` is an iterable then freeze() is called on all items of the @@ -102,6 +114,8 @@ KIND_AS_STR = 1 KIND_AS_MPY = 2 KIND_MPY = 3 +VARS = {} + manifest_list = [] class FreezeError(Exception): @@ -115,7 +129,12 @@ def system(cmd): return -1, er.output def convert_path(path): - return path.replace('$(MPY)', TOP) + # Perform variable substituion. + for name, value in VARS.items(): + path = path.replace('$({})'.format(name), value) + # Convert to absolute path (so that future operations don't rely on + # still being chdir'ed). + return os.path.abspath(path) def get_timestamp(path, default=None): try: @@ -173,23 +192,39 @@ def freeze_internal(kind, path, script, opt): manifest_list.append((kind, path, script, opt)) def main(): - global TOP - # Parse arguments - assert sys.argv[1] == '-o' - output_file = sys.argv[2] - TOP = sys.argv[3] - BUILD = sys.argv[4] - mpy_cross_flags = sys.argv[5] - input_manifest_list = sys.argv[6:] + import argparse + cmd_parser = argparse.ArgumentParser(description='A tool to generate frozen content in MicroPython firmware images.') + cmd_parser.add_argument('-o', '--output', help='output path') + cmd_parser.add_argument('-b', '--build-dir', help='output path') + cmd_parser.add_argument('-f', '--mpy-cross-flags', default='', help='flags to pass to mpy-cross') + cmd_parser.add_argument('-v', '--var', action='append', help='variables to substitute') + cmd_parser.add_argument('files', nargs='+', help='input manifest list') + args = cmd_parser.parse_args() + + # Extract variables for substitution. + for var in args.var: + name, value = var.split('=', 1) + if os.path.exists(value): + value = os.path.abspath(value) + VARS[name] = value + + if 'MPY_DIR' not in VARS or 'PORT_DIR' not in VARS: + print('MPY_DIR and PORT_DIR variables must be specified') + sys.exit(1) # Get paths to tools - MAKE_FROZEN = TOP + '/tools/make-frozen.py' - MPY_CROSS = TOP + '/mpy-cross/mpy-cross' - MPY_TOOL = TOP + '/tools/mpy-tool.py' + MAKE_FROZEN = VARS['MPY_DIR'] + '/tools/make-frozen.py' + MPY_CROSS = VARS['MPY_DIR'] + '/mpy-cross/mpy-cross' + MPY_TOOL = VARS['MPY_DIR'] + '/tools/mpy-tool.py' + + # Ensure mpy-cross is built + if not os.path.exists(MPY_CROSS): + print('mpy-cross not found at {}, please build it first'.format(MPY_CROSS)) + sys.exit(1) # Include top-level inputs, to generate the manifest - for input_manifest in input_manifest_list: + for input_manifest in args.files: try: if input_manifest.endswith('.py'): include(input_manifest) @@ -209,13 +244,13 @@ def main(): ts_outfile = get_timestamp_newest(path) elif kind == KIND_AS_MPY: infile = '{}/{}'.format(path, script) - outfile = '{}/frozen_mpy/{}.mpy'.format(BUILD, script[:-3]) + outfile = '{}/frozen_mpy/{}.mpy'.format(args.build_dir, script[:-3]) ts_infile = get_timestamp(infile) ts_outfile = get_timestamp(outfile, 0) if ts_infile >= ts_outfile: print('MPY', script) mkdir(outfile) - res, out = system([MPY_CROSS] + mpy_cross_flags.split() + ['-o', outfile, '-s', script, '-O{}'.format(opt), infile]) + res, out = system([MPY_CROSS] + args.mpy_cross_flags.split() + ['-o', outfile, '-s', script, '-O{}'.format(opt), infile]) if res != 0: print('error compiling {}: {}'.format(infile, out)) raise SystemExit(1) @@ -229,20 +264,26 @@ def main(): ts_newest = max(ts_newest, ts_outfile) # Check if output file needs generating - if ts_newest < get_timestamp(output_file, 0): + if ts_newest < get_timestamp(args.output, 0): # No files are newer than output file so it does not need updating return # Freeze paths as strings - _, output_str = system([MAKE_FROZEN] + str_paths) + res, output_str = system([MAKE_FROZEN] + str_paths) + if res != 0: + print('error freezing strings {}: {}'.format(str_paths, output_str)) + sys.exit(1) # Freeze .mpy files - _, output_mpy = system([MPY_TOOL, '-f', '-q', BUILD + '/genhdr/qstrdefs.preprocessed.h'] + mpy_files) + res, output_mpy = system([MPY_TOOL, '-f', '-q', args.build_dir + '/genhdr/qstrdefs.preprocessed.h'] + mpy_files) + if res != 0: + print('error freezing mpy {}: {}'.format(mpy_files, output_mpy)) + sys.exit(1) # Generate output - print('GEN', output_file) - mkdir(output_file) - with open(output_file, 'wb') as f: + print('GEN', args.output) + mkdir(args.output) + with open(args.output, 'wb') as f: f.write(b'//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n') f.write(output_str) f.write(b'//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n')