From 29efbb24d992564db4bbb808597719934ed9ac9f Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 24 Sep 2019 16:29:58 -0700 Subject: [PATCH] docs: Use make invocation's -j argument for parallelism While sphinx 1.7 and later supports "-jauto" for parallelism, this effectively ignores the "-j" flag used in the "make" invocation, which may cause confusion for build systems. Instead, extract the available parallelism from "make"'s job server (since it is not exposed in any special variables) and use that for the "sphinx-build" run. Now things work correctly for builds where -j is specified at the top-level: make -j16 htmldocs If -j is not specified, continue to fallback to "-jauto" if available. Signed-off-by: Kees Cook Signed-off-by: Jonathan Corbet --- Documentation/Makefile | 3 ++- scripts/jobserver-count | 58 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100755 scripts/jobserver-count diff --git a/Documentation/Makefile b/Documentation/Makefile index e145e4db508b..c6e564656a5b 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -33,7 +33,7 @@ ifeq ($(HAVE_SPHINX),0) else # HAVE_SPHINX -export SPHINXOPTS = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "-jauto" if ($$1 >= "1.7") } ;} close IN') +export SPHINX_PARALLEL = $(shell perl -e 'open IN,"sphinx-build --version 2>&1 |"; while () { if (m/([\d\.]+)/) { print "auto" if ($$1 >= "1.7") } ;} close IN') # User-friendly check for pdflatex and latexmk HAVE_PDFLATEX := $(shell if which $(PDFLATEX) >/dev/null 2>&1; then echo 1; else echo 0; fi) @@ -68,6 +68,7 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) PYTHONDONTWRITEBYTECODE=1 \ BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ $(SPHINXBUILD) \ + -j $(shell python $(srctree)/scripts/jobserver-count $(SPHINX_PARALLEL)) \ -b $2 \ -c $(abspath $(srctree)/$(src)) \ -d $(abspath $(BUILDDIR)/.doctrees/$3) \ diff --git a/scripts/jobserver-count b/scripts/jobserver-count new file mode 100755 index 000000000000..0b482d6884d2 --- /dev/null +++ b/scripts/jobserver-count @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0+ +# +# This determines how many parallel tasks "make" is expecting, as it is +# not exposed via an special variables. +# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html#POSIX-Jobserver +from __future__ import print_function +import os, sys, fcntl, errno + +# Default parallelism is "1" unless overridden on the command-line. +default="1" +if len(sys.argv) > 1: + default=sys.argv[1] + +# Set non-blocking for a given file descriptor. +def nonblock(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + return fd + +# Extract and prepare jobserver file descriptors from envirnoment. +try: + # Fetch the make environment options. + flags = os.environ['MAKEFLAGS'] + + # Look for "--jobserver=R,W" + opts = [x for x in flags.split(" ") if x.startswith("--jobserver")] + + # Parse out R,W file descriptor numbers and set them nonblocking. + fds = opts[0].split("=", 1)[1] + reader, writer = [int(x) for x in fds.split(",", 1)] + reader = nonblock(reader) +except (KeyError, IndexError, ValueError, IOError): + # Any missing environment strings or bad fds should result in just + # using the default specified parallelism. + print(default) + sys.exit(0) + +# Read out as many jobserver slots as possible. +jobs = b"" +while True: + try: + slot = os.read(reader, 1) + jobs += slot + except (OSError, IOError) as e: + if e.errno == errno.EWOULDBLOCK: + # Stop when reach the end of the jobserver queue. + break + raise e +# Return all the reserved slots. +os.write(writer, jobs) + +# If the jobserver was (impossibly) full or communication failed, use default. +if len(jobs) < 1: + print(default) + +# Report available slots (with a bump for our caller's reserveration). +print(len(jobs) + 1)