py/makeqstrdefs.py: Support preprocessing C++ files for QSTR generation.

When SCR_QSTR contains C++ files they should be preprocessed with the same
compiler flags (CXXFLAGS) as they will be compiled with, to make sure code
scanned for QSTR occurrences is effectively the code used in the rest of
the build.  The 'split SCR_QSTR in .c and .cpp files and process each with
different flags' logic isn't trivial to express in a Makefile and the
existing principle for deciding which files to preprocess was already
rather complicated, so the actual preprocessing is moved into
makeqstrdefs.py completely.

py/makeqstrdefs.py

@@ -8,6 +8,7 @@ This script works with Python 2.6, 2.7, 3.3 and 3.4.
 from __future__ import print_function
 
 import re
+import subprocess
 import sys
 import io
 import os
@@ -20,6 +21,31 @@ _MODE_QSTR = "qstr"
 _MODE_COMPRESS = "compress"
 
 
+def preprocess():
+    if any(src in args.dependencies for src in args.changed_sources):
+        sources = args.sources
+    elif any(args.changed_sources):
+        sources = args.changed_sources
+    else:
+        sources = args.sources
+    csources = []
+    cxxsources = []
+    for source in sources:
+        if source.endswith(".cpp"):
+            cxxsources.append(source)
+        else:
+            csources.append(source)
+    try:
+        os.makedirs(os.path.dirname(args.output[0]))
+    except OSError:
+        pass
+    with open(args.output[0], "w") as out_file:
+        if csources:
+            subprocess.check_call(args.pp + args.cflags + csources, stdout=out_file)
+        if cxxsources:
+            subprocess.check_call(args.pp + args.cxxflags + cxxsources, stdout=out_file)
+
+
 def write_out(fname, output):
     if output:
         for m, r in [("/", "__"), ("\\", "__"), (":", "@"), ("..", "@@")]:
@@ -105,7 +131,7 @@ def cat_together():
 
 
 if __name__ == "__main__":
-    if len(sys.argv) != 6:
+    if len(sys.argv) < 6:
         print("usage: %s command mode input_filename output_dir output_file" % sys.argv[0])
         sys.exit(2)
 
@@ -114,6 +140,37 @@ if __name__ == "__main__":
 
     args = Args()
     args.command = sys.argv[1]
+
+    if args.command == "pp":
+        named_args = {
+            s: []
+            for s in [
+                "pp",
+                "output",
+                "cflags",
+                "cxxflags",
+                "sources",
+                "changed_sources",
+                "dependencies",
+            ]
+        }
+
+        for arg in sys.argv[1:]:
+            if arg in named_args:
+                current_tok = arg
+            else:
+                named_args[current_tok].append(arg)
+
+        if not named_args["pp"] or len(named_args["output"]) != 1:
+            print("usage: %s %s ..." % (sys.argv[0], " ... ".join(named_args)))
+            sys.exit(2)
+
+        for k, v in named_args.items():
+            setattr(args, k, v)
+
+        preprocess()
+        sys.exit(0)
+
     args.mode = sys.argv[2]
     args.input_filename = sys.argv[3]  # Unused for command=cat
     args.output_dir = sys.argv[4]

py/mkrules.mk

@@ -15,10 +15,12 @@ CFLAGS += -DMICROPY_ROM_TEXT_COMPRESSION=1
 endif
 
 # QSTR generation uses the same CFLAGS, with these modifications.
+QSTR_GEN_FLAGS = -DNO_QSTR -I$(BUILD)/tmp
 # Note: := to force evalulation immediately.
 QSTR_GEN_CFLAGS := $(CFLAGS)
-QSTR_GEN_CFLAGS += -DNO_QSTR
-QSTR_GEN_CFLAGS += -I$(BUILD)/tmp
+QSTR_GEN_CFLAGS += $(QSTR_GEN_FLAGS)
+QSTR_GEN_CXXFLAGS := $(CXXFLAGS)
+QSTR_GEN_CXXFLAGS += $(QSTR_GEN_FLAGS)
 
 # This file expects that OBJ contains a list of all of the object files.
 # The directory portion of each object file is used to locate the source
@@ -95,14 +97,14 @@ $(BUILD)/%.pp: %.c
 # to get built before we try to compile any of them.
 $(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h $(OBJ_EXTRA_ORDER_DEPS)
 
-# The logic for qstr regeneration is:
+# The logic for qstr regeneration (applied by makeqstrdefs.py) is:
 # - if anything in QSTR_GLOBAL_DEPENDENCIES is newer, then process all source files ($^)
 # - else, if list of newer prerequisites ($?) is not empty, then process just these ($?)
 # - else, process all source files ($^) [this covers "make -B" which can set $? to empty]
 # See more information about this process in docs/develop/qstr.rst.
 $(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) $(QSTR_GLOBAL_DEPENDENCIES) | $(QSTR_GLOBAL_REQUIREMENTS)
 	$(ECHO) "GEN $@"
-	$(Q)$(CPP) $(QSTR_GEN_CFLAGS) $(if $(filter $?,$(QSTR_GLOBAL_DEPENDENCIES)),$^,$(if $?,$?,$^)) >$(HEADER_BUILD)/qstr.i.last
+	$(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py pp $(CPP) output $(HEADER_BUILD)/qstr.i.last cflags $(QSTR_GEN_CFLAGS) cxxflags $(QSTR_GEN_CXXFLAGS) sources $^ dependencies $(QSTR_GLOBAL_DEPENDENCIES) changed_sources $?
 
 $(HEADER_BUILD)/qstr.split: $(HEADER_BUILD)/qstr.i.last
 	$(ECHO) "GEN $@"