tools/makemanifest.py: Eval relative paths w.r.t. current manifest file.

When loading a manifest file, e.g. by include(), it will chdir first to the
directory of that manifest.  This means that all file operations within a
manifest are relative to that manifest's location.

As a consequence of this, additional environment variables are needed to
find absolute paths, so the following are added: $(MPY_LIB_DIR),
$(PORT_DIR), $(BOARD_DIR).  And rename $(MPY) to $(MPY_DIR) to be
consistent.

Existing manifests are updated to match.

ports/esp32/boards/manifest.py

@@ -1,6 +1,6 @@
-freeze('modules')
+freeze('$(PORT_DIR)/modules')
-freeze('$(MPY)/tools', ('upip.py', 'upip_utarfile.py'))
+freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py'))
-freeze('$(MPY)/ports/esp8266/modules', 'ntptime.py')
+freeze('$(MPY_DIR)/ports/esp8266/modules', 'ntptime.py')
-freeze('$(MPY)/ports/esp8266/modules', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',))
+freeze('$(MPY_DIR)/ports/esp8266/modules', ('webrepl.py', 'webrepl_setup.py', 'websocket_helper.py',))
-freeze('$(MPY)/drivers/dht', 'dht.py')
+freeze('$(MPY_DIR)/drivers/dht', 'dht.py')
-freeze('$(MPY)/drivers/onewire')
+freeze('$(MPY_DIR)/drivers/onewire')

ports/esp32/boards/manifest_release.py

@@ -1,8 +1,6 @@
-include('boards/manifest.py')
+include('manifest.py')
 
-LIB = '../../../micropython-lib'
+freeze('$(MPY_LIB_DIR)/upysh', 'upysh.py')
-
+freeze('$(MPY_LIB_DIR)/urequests', 'urequests.py')
-freeze(LIB + '/upysh', 'upysh.py')
+freeze('$(MPY_LIB_DIR)/umqtt.simple', 'umqtt/simple.py')
-freeze(LIB + '/urequests', 'urequests.py')
+freeze('$(MPY_LIB_DIR)/umqtt.robust', 'umqtt/robust.py')
-freeze(LIB + '/umqtt.simple', 'umqtt/simple.py')
-freeze(LIB + '/umqtt.robust', 'umqtt/robust.py')

ports/esp8266/boards/manifest.py

@@ -1,4 +1,4 @@
-freeze('modules')
+freeze('$(PORT_DIR)/modules')
-freeze('$(MPY)/tools', ('upip.py', 'upip_utarfile.py'))
+freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py'))
-freeze('$(MPY)/drivers/dht', 'dht.py')
+freeze('$(MPY_DIR)/drivers/dht', 'dht.py')
-freeze('$(MPY)/drivers/onewire')
+freeze('$(MPY_DIR)/drivers/onewire')

ports/stm32/boards/manifest.py

@@ -1,3 +1,3 @@
-freeze('$(MPY)/drivers/dht', 'dht.py')
+freeze('$(MPY_DIR)/drivers/dht', 'dht.py')
-freeze('$(MPY)/drivers/display', ('lcd160cr.py', 'lcd160cr_test.py'))
+freeze('$(MPY_DIR)/drivers/display', ('lcd160cr.py', 'lcd160cr_test.py'))
-freeze('$(MPY)/drivers/onewire', 'onewire.py')
+freeze('$(MPY_DIR)/drivers/onewire', 'onewire.py')

ports/unix/manifest.py

@@ -1,2 +1,2 @@
-freeze_as_mpy('$(MPY)/tools', 'upip.py')
+freeze_as_mpy('$(MPY_DIR)/tools', 'upip.py')
-freeze_as_mpy('$(MPY)/tools', 'upip_utarfile.py', opt=3)
+freeze_as_mpy('$(MPY_DIR)/tools', 'upip_utarfile.py', opt=3)

py/mkenv.mk

@@ -66,6 +66,8 @@ MAKE_FROZEN = $(PYTHON) $(TOP)/tools/make-frozen.py
 MPY_CROSS = $(TOP)/mpy-cross/mpy-cross
 MPY_TOOL = $(PYTHON) $(TOP)/tools/mpy-tool.py
 
+MPY_LIB_DIR = $(TOP)/../micropython-lib
+
 all:
 .PHONY: all
 

py/mkrules.mk

@@ -100,7 +100,7 @@ $(HEADER_BUILD):
 ifneq ($(FROZEN_MANIFEST),)
 # to build frozen_content.c from a manifest
 $(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h
-	$(Q)$(MAKE_MANIFEST) -o $@ $(TOP) $(BUILD) "$(MPY_CROSS_FLAGS)" $(FROZEN_MANIFEST)
+	$(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) $(FROZEN_MANIFEST)
 endif
 
 ifneq ($(FROZEN_DIR),)

tools/makemanifest.py

@@ -38,14 +38,22 @@ def include(manifest):
 
     The manifest argument can be a string (filename) or an iterable of
     strings.
+
+    Relative paths are resolved with respect to the current manifest file.
     """
 
     if not isinstance(manifest, str):
         for m in manifest:
             include(m)
     else:
+        manifest = convert_path(manifest)
         with open(manifest) as f:
+            # Make paths relative to this manifest file while processing it.
+            # Applies to includes and input files.
+            prev_cwd = os.getcwd()
+            os.chdir(os.path.dirname(manifest))
             exec(f.read())
+            os.chdir(prev_cwd)
 
 def freeze(path, script=None, opt=0):
     """Freeze the input, automatically determining its type.  A .py script
@@ -57,6 +65,10 @@ def freeze(path, script=None, opt=0):
     the module will start after `path`, ie `path` is excluded from the
     module name.
 
+    If `path` is relative, it is resolved to the current manifest.py.
+    Use $(MPY_DIR), $(MPY_LIB_DIR), $(PORT_DIR), $(BOARD_DIR) if you need
+    to access specific paths.
+
     If `script` is None all files in `path` will be frozen.
 
     If `script` is an iterable then freeze() is called on all items of the
@@ -102,6 +114,8 @@ KIND_AS_STR = 1
 KIND_AS_MPY = 2
 KIND_MPY = 3
 
+VARS = {}
+
 manifest_list = []
 
 class FreezeError(Exception):
@@ -115,7 +129,12 @@ def system(cmd):
         return -1, er.output
 
 def convert_path(path):
-    return path.replace('$(MPY)', TOP)
+    # Perform variable substituion.
+    for name, value in VARS.items():
+        path = path.replace('$({})'.format(name), value)
+    # Convert to absolute path (so that future operations don't rely on
+    # still being chdir'ed).
+    return os.path.abspath(path)
 
 def get_timestamp(path, default=None):
     try:
@@ -173,23 +192,39 @@ def freeze_internal(kind, path, script, opt):
         manifest_list.append((kind, path, script, opt))
 
 def main():
-    global TOP
-
     # Parse arguments
-    assert sys.argv[1] == '-o'
+    import argparse
-    output_file = sys.argv[2]
+    cmd_parser = argparse.ArgumentParser(description='A tool to generate frozen content in MicroPython firmware images.')
-    TOP = sys.argv[3]
+    cmd_parser.add_argument('-o', '--output', help='output path')
-    BUILD = sys.argv[4]
+    cmd_parser.add_argument('-b', '--build-dir', help='output path')
-    mpy_cross_flags = sys.argv[5]
+    cmd_parser.add_argument('-f', '--mpy-cross-flags', default='', help='flags to pass to mpy-cross')
-    input_manifest_list = sys.argv[6:]
+    cmd_parser.add_argument('-v', '--var', action='append', help='variables to substitute')
+    cmd_parser.add_argument('files', nargs='+', help='input manifest list')
+    args = cmd_parser.parse_args()
+
+    # Extract variables for substitution.
+    for var in args.var:
+        name, value = var.split('=', 1)
+        if os.path.exists(value):
+            value = os.path.abspath(value)
+        VARS[name] = value
+
+    if 'MPY_DIR' not in VARS or 'PORT_DIR' not in VARS:
+        print('MPY_DIR and PORT_DIR variables must be specified')
+        sys.exit(1)
 
     # Get paths to tools
-    MAKE_FROZEN = TOP + '/tools/make-frozen.py'
+    MAKE_FROZEN = VARS['MPY_DIR'] + '/tools/make-frozen.py'
-    MPY_CROSS = TOP + '/mpy-cross/mpy-cross'
+    MPY_CROSS = VARS['MPY_DIR'] + '/mpy-cross/mpy-cross'
-    MPY_TOOL = TOP + '/tools/mpy-tool.py'
+    MPY_TOOL = VARS['MPY_DIR'] + '/tools/mpy-tool.py'
+
+    # Ensure mpy-cross is built
+    if not os.path.exists(MPY_CROSS):
+        print('mpy-cross not found at {}, please build it first'.format(MPY_CROSS))
+        sys.exit(1)
 
     # Include top-level inputs, to generate the manifest
-    for input_manifest in input_manifest_list:
+    for input_manifest in args.files:
         try:
             if input_manifest.endswith('.py'):
                 include(input_manifest)
@@ -209,13 +244,13 @@ def main():
             ts_outfile = get_timestamp_newest(path)
         elif kind == KIND_AS_MPY:
             infile = '{}/{}'.format(path, script)
-            outfile = '{}/frozen_mpy/{}.mpy'.format(BUILD, script[:-3])
+            outfile = '{}/frozen_mpy/{}.mpy'.format(args.build_dir, script[:-3])
             ts_infile = get_timestamp(infile)
             ts_outfile = get_timestamp(outfile, 0)
             if ts_infile >= ts_outfile:
                 print('MPY', script)
                 mkdir(outfile)
-                res, out = system([MPY_CROSS] + mpy_cross_flags.split() + ['-o', outfile, '-s', script, '-O{}'.format(opt), infile])
+                res, out = system([MPY_CROSS] + args.mpy_cross_flags.split() + ['-o', outfile, '-s', script, '-O{}'.format(opt), infile])
                 if res != 0:
                     print('error compiling {}: {}'.format(infile, out))
                     raise SystemExit(1)
@@ -229,20 +264,26 @@ def main():
         ts_newest = max(ts_newest, ts_outfile)
 
     # Check if output file needs generating
-    if ts_newest < get_timestamp(output_file, 0):
+    if ts_newest < get_timestamp(args.output, 0):
         # No files are newer than output file so it does not need updating
         return
 
     # Freeze paths as strings
-    _, output_str = system([MAKE_FROZEN] + str_paths)
+    res, output_str = system([MAKE_FROZEN] + str_paths)
+    if res != 0:
+        print('error freezing strings {}: {}'.format(str_paths, output_str))
+        sys.exit(1)
 
     # Freeze .mpy files
-    _, output_mpy = system([MPY_TOOL, '-f', '-q', BUILD + '/genhdr/qstrdefs.preprocessed.h'] + mpy_files)
+    res, output_mpy = system([MPY_TOOL, '-f', '-q', args.build_dir + '/genhdr/qstrdefs.preprocessed.h'] + mpy_files)
+    if res != 0:
+        print('error freezing mpy {}: {}'.format(mpy_files, output_mpy))
+        sys.exit(1)
 
     # Generate output
-    print('GEN', output_file)
+    print('GEN', args.output)
-    mkdir(output_file)
+    mkdir(args.output)
-    with open(output_file, 'wb') as f:
+    with open(args.output, 'wb') as f:
         f.write(b'//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n')
         f.write(output_str)
         f.write(b'//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n')