From a7a0729c4532c94f4e8efb4faaf6ef00e5fe19ba Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 12 Aug 2016 14:11:12 -0600 Subject: [PATCH 01/51] docs: Set the Sphinx default highlight language to "guess" This should eliminate a whole class of markup warnings, at the cost of occasionally amusing markup choices; we'll have to see if it works out. Suggested-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 96b7aa66c89c..106ae9c740b9 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -131,7 +131,7 @@ pygments_style = 'sphinx' todo_include_todos = False primary_domain = 'C' -highlight_language = 'C' +highlight_language = 'guess' # -- Options for HTML output ---------------------------------------------- From c2d5be14cb39da34b076b461d8aa2344611364c6 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Fri, 12 Aug 2016 15:12:36 -0600 Subject: [PATCH 02/51] docs: kernel-documentation: remove some highlight directives With the conf.py change, we don't need them to avoid warnings anymore. Signed-off-by: Jonathan Corbet --- Documentation/kernel-documentation.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Documentation/kernel-documentation.rst b/Documentation/kernel-documentation.rst index c4eb5049da39..391decc66a18 100644 --- a/Documentation/kernel-documentation.rst +++ b/Documentation/kernel-documentation.rst @@ -366,8 +366,6 @@ Domain`_ references. Cross-referencing from reStructuredText ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. highlight:: none - To cross-reference the functions and types defined in the kernel-doc comments from reStructuredText documents, please use the `Sphinx C Domain`_ references. For example:: @@ -390,8 +388,6 @@ For further details, please refer to the `Sphinx C Domain`_ documentation. Function documentation ---------------------- -.. highlight:: c - The general format of a function and function-like macro kernel-doc comment is:: /** @@ -572,8 +568,6 @@ DocBook XML [DEPRECATED] Converting DocBook to Sphinx ---------------------------- -.. highlight:: none - Over time, we expect all of the documents under ``Documentation/DocBook`` to be converted to Sphinx and reStructuredText. For most DocBook XML documents, a good enough solution is to use the simple ``Documentation/sphinx/tmplcvt`` script, From 609afe6b49ef3b4b58886f7b82013a2f35f59a57 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 10 Aug 2016 18:54:07 +0300 Subject: [PATCH 03/51] Documentation/sphinx: build the media intermediate rst files for all outputs This is a stopgap measure to allow building outputs other than html. Signed-off-by: Jani Nikula Acked-by: Mauro Carvalho Chehab Acked-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/Makefile.sphinx | 3 +-- Documentation/media/Makefile | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index 857f1e273418..d24a9f12a280 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -38,10 +38,9 @@ ALLSPHINXOPTS = -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) -d $(B I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . quiet_cmd_sphinx = SPHINX $@ - cmd_sphinx = BUILDDIR=$(BUILDDIR) $(SPHINXBUILD) -b $2 $(ALLSPHINXOPTS) $(BUILDDIR)/$2 + cmd_sphinx = $(MAKE) BUILDDIR=$(BUILDDIR) $(build)=Documentation/media all; BUILDDIR=$(BUILDDIR) $(SPHINXBUILD) -b $2 $(ALLSPHINXOPTS) $(BUILDDIR)/$2 htmldocs: - $(MAKE) BUILDDIR=$(BUILDDIR) -f $(srctree)/Documentation/media/Makefile $@ $(call cmd,sphinx,html) pdfdocs: diff --git a/Documentation/media/Makefile b/Documentation/media/Makefile index 39e2d766dbe3..a7fb35291f6c 100644 --- a/Documentation/media/Makefile +++ b/Documentation/media/Makefile @@ -10,7 +10,8 @@ FILES = audio.h.rst ca.h.rst dmx.h.rst frontend.h.rst net.h.rst video.h.rst \ TARGETS := $(addprefix $(BUILDDIR)/, $(FILES)) -htmldocs: $(BUILDDIR) ${TARGETS} +.PHONY: all +all: $(BUILDDIR) ${TARGETS} $(BUILDDIR): $(Q)mkdir -p $@ From f907ba930780d73e87791c73a4fdc1998e59db75 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 10 Aug 2016 18:54:08 +0300 Subject: [PATCH 04/51] Documentation: switch to pdflatex for pdf generation Looks like rst2pdf is not robust enough, especially for large documents. Use recursive make on the Sphinx generated makefile to convert latex to pdf. The ugly detail is that pdf is generated into Documentation/output/latex. Unfortunately, the pdflatex build generates huge amounts of build log noise, and also fails in the end. We'll fix that next. Signed-off-by: Jani Nikula Acked-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/Makefile.sphinx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index d24a9f12a280..fc29e08085aa 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -25,8 +25,8 @@ else ifneq ($(DOCBOOKS),) else # HAVE_SPHINX -# User-friendly check for rst2pdf -HAVE_RST2PDF := $(shell if python -c "import rst2pdf" >/dev/null 2>&1; then echo 1; else echo 0; fi) +# User-friendly check for pdflatex +HAVE_PDFLATEX := $(shell if which pdflatex >/dev/null 2>&1; then echo 1; else echo 0; fi) # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 @@ -44,12 +44,13 @@ htmldocs: $(call cmd,sphinx,html) pdfdocs: -ifeq ($(HAVE_RST2PDF),0) - $(warning The Python 'rst2pdf' module was not found. Make sure you have the module installed to produce PDF output.) +ifeq ($(HAVE_PDFLATEX),0) + $(warning The 'pdflatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) @echo " SKIP Sphinx $@ target." -else # HAVE_RST2PDF - $(call cmd,sphinx,pdf) -endif # HAVE_RST2PDF +else # HAVE_PDFLATEX + $(call cmd,sphinx,latex) + $(Q)$(MAKE) -C $(BUILDDIR)/latex +endif # HAVE_PDFLATEX epubdocs: $(call cmd,sphinx,epub) From 3eb6cd6834c356f40e1633a0ced4ff9a4c59936b Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 10 Aug 2016 18:54:09 +0300 Subject: [PATCH 05/51] Documentation: exclude media documentation from pdf generation Although pdflatex is more robust than rst2pdf, building media documentation pdf still fails. Exclude media documentation from pdf generation for now. Signed-off-by: Jani Nikula Acked-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 96b7aa66c89c..827dafc515b1 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -268,7 +268,9 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'TheLinuxKernel.tex', 'The Linux Kernel Documentation', + ('kernel-documentation', 'kernel-documentation.tex', 'The Linux Kernel Documentation', + 'The kernel development community', 'manual'), + ('gpu/index', 'gpu.tex', 'Linux GPU Driver Developer\'s Guide', 'The kernel development community', 'manual'), ] From 606b9ac81a63ab3adb7e61206b9ae34ee186a89d Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 13 Aug 2016 16:12:42 +0200 Subject: [PATCH 06/51] doc-rst: generic way to build only sphinx sub-folders Add a generic way to build only a reST sub-folder with or without a individual *build-theme*. * control *sub-folders* by environment SPHINXDIRS * control *build-theme* by environment SPHINX_CONF Folders with a conf.py file, matching $(srctree)/Documentation/*/conf.py can be build and distributed *stand-alone*. E.g. to compile only the html of 'media' and 'gpu' folder use:: make SPHINXDIRS="media gpu" htmldocs To use an additional sphinx-build configuration (*build-theme*) set the name of the configuration file to SPHINX_CONF. E.g. to compile only the html of 'media' with the *nit-picking* build use:: make SPHINXDIRS=media SPHINX_CONF=conf_nitpick.py htmldocs With this, the Documentation/conf.py is read first and updated with the configuration values from the Documentation/media/conf_nitpick.py. Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/DocBook/Makefile | 9 +++++- Documentation/Makefile.sphinx | 43 ++++++++++++++++++++++++----- Documentation/conf.py | 7 +++++ Documentation/sphinx/load_config.py | 32 +++++++++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 Documentation/sphinx/load_config.py diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile index 64460a897f56..a91c96522379 100644 --- a/Documentation/DocBook/Makefile +++ b/Documentation/DocBook/Makefile @@ -22,8 +22,14 @@ ifeq ($(DOCBOOKS),) # Skip DocBook build if the user explicitly requested no DOCBOOKS. .DEFAULT: @echo " SKIP DocBook $@ target (DOCBOOKS=\"\" specified)." - else +ifneq ($(SPHINXDIRS),) + +# Skip DocBook build if the user explicitly requested a sphinx dir +.DEFAULT: + @echo " SKIP DocBook $@ target (SPHINXDIRS specified)." +else + ### # The build process is as follows (targets): @@ -221,6 +227,7 @@ silent_gen_xml = : echo "") > $@ endif # DOCBOOKS="" +endif # SPHINDIR=... ### # Help targets as used by the top-level makefile diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index fc29e08085aa..ea0664cece12 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -5,6 +5,9 @@ # You can set these variables from the command line. SPHINXBUILD = sphinx-build SPHINXOPTS = +SPHINXDIRS = . +_SPHINXDIRS = $(patsubst $(srctree)/Documentation/%/conf.py,%,$(wildcard $(srctree)/Documentation/*/conf.py)) +SPHINX_CONF = conf.py PAPER = BUILDDIR = $(obj)/output @@ -33,30 +36,50 @@ PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter KERNELDOC = $(srctree)/scripts/kernel-doc KERNELDOC_CONF = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) -ALLSPHINXOPTS = -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) -d $(BUILDDIR)/.doctrees $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) +ALLSPHINXOPTS = $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -quiet_cmd_sphinx = SPHINX $@ - cmd_sphinx = $(MAKE) BUILDDIR=$(BUILDDIR) $(build)=Documentation/media all; BUILDDIR=$(BUILDDIR) $(SPHINXBUILD) -b $2 $(ALLSPHINXOPTS) $(BUILDDIR)/$2 +# commands; the 'cmd' from scripts/Kbuild.include is not *loopable* +loop_cmd = $(echo-cmd) $(cmd_$(1)) + +# $2 sphinx builder e.g. "html" +# $3 name of the build subfolder / e.g. "media", used as: +# * dest folder relative to $(BUILDDIR) and +# * cache folder relative to $(BUILDDIR)/.doctrees +# $4 dest subfolder e.g. "man" for man pages at media/man +# $5 reST source folder relative to $(srctree)/$(src), +# e.g. "media" for the linux-tv book-set at ./Documentation/media + +quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4); + cmd_sphinx = $(MAKE) BUILDDIR=$(abspath $(BUILDDIR)) $(build)=Documentation/media all;\ + BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ + $(SPHINXBUILD) \ + -b $2 \ + -c $(abspath $(srctree)/$(src)) \ + -d $(abspath $(BUILDDIR)/.doctrees/$3) \ + -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) \ + $(ALLSPHINXOPTS) \ + $(abspath $(srctree)/$(src)/$5) \ + $(abspath $(BUILDDIR)/$3/$4); htmldocs: - $(call cmd,sphinx,html) + @$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var))) pdfdocs: ifeq ($(HAVE_PDFLATEX),0) $(warning The 'pdflatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) @echo " SKIP Sphinx $@ target." else # HAVE_PDFLATEX - $(call cmd,sphinx,latex) + @$(call loop_cmd,sphinx,latex,.,latex,.)) $(Q)$(MAKE) -C $(BUILDDIR)/latex endif # HAVE_PDFLATEX epubdocs: - $(call cmd,sphinx,epub) + @$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,epub,$(var),epub,$(var))) xmldocs: - $(call cmd,sphinx,xml) + @$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,xml,$(var),xml,$(var))) # no-ops for the Sphinx toolchain sgmldocs: @@ -76,3 +99,9 @@ dochelp: @echo ' epubdocs - EPUB' @echo ' xmldocs - XML' @echo ' cleandocs - clean all generated files' + @echo + @echo ' make SPHINXDIRS="s1 s2" [target] Generate only docs of folder s1, s2' + @echo ' valid values for SPHINXDIRS are: $(_SPHINXDIRS)' + @echo + @echo ' make SPHINX_CONF={conf-file} [target] use *additional* sphinx-build' + @echo ' configuration. This is e.g. useful to build with nit-picking config.' diff --git a/Documentation/conf.py b/Documentation/conf.py index 827dafc515b1..43e41ed74584 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -19,6 +19,7 @@ import os # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('sphinx')) +from load_config import loadConfig # -- General configuration ------------------------------------------------ @@ -421,3 +422,9 @@ pdf_documents = [ # line arguments. kerneldoc_bin = '../scripts/kernel-doc' kerneldoc_srctree = '..' + +# ------------------------------------------------------------------------------ +# Since loadConfig overwrites settings from the global namespace, it has to be +# the last statement in the conf.py file +# ------------------------------------------------------------------------------ +loadConfig(globals()) diff --git a/Documentation/sphinx/load_config.py b/Documentation/sphinx/load_config.py new file mode 100644 index 000000000000..301a21aa4f63 --- /dev/null +++ b/Documentation/sphinx/load_config.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8; mode: python -*- +# pylint: disable=R0903, C0330, R0914, R0912, E0401 + +import os +import sys +from sphinx.util.pycompat import execfile_ + +# ------------------------------------------------------------------------------ +def loadConfig(namespace): +# ------------------------------------------------------------------------------ + + u"""Load an additional configuration file into *namespace*. + + The name of the configuration file is taken from the environment + ``SPHINX_CONF``. The external configuration file extends (or overwrites) the + configuration values from the origin ``conf.py``. With this you are able to + maintain *build themes*. """ + + config_file = os.environ.get("SPHINX_CONF", None) + if (config_file is not None + and os.path.normpath(namespace["__file__"]) != os.path.normpath(config_file) ): + config_file = os.path.abspath(config_file) + + if os.path.isfile(config_file): + sys.stdout.write("load additional sphinx-config: %s\n" % config_file) + config = namespace.copy() + config['__file__'] = config_file + execfile_(config_file, config) + del config['__file__'] + namespace.update(config) + else: + sys.stderr.write("WARNING: additional sphinx-config not found: %s\n" % config_file) From b32febad77068b4a28daf7b7e063438d0cca8a42 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 13 Aug 2016 16:12:43 +0200 Subject: [PATCH 07/51] doc-rst: add stand-alone conf.py to media folder With the media/conf.py, and media/index.rst the media folder can be build and distributed stand-alone. Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/index.rst | 7 +------ Documentation/media/conf.py | 3 +++ Documentation/media/index.rst | 12 ++++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 Documentation/media/conf.py create mode 100644 Documentation/media/index.rst diff --git a/Documentation/index.rst b/Documentation/index.rst index e0fc72963e87..bdd9525e05aa 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -6,18 +6,13 @@ Welcome to The Linux Kernel's documentation! ============================================ -Nothing for you to see here *yet*. Please move along. - Contents: .. toctree:: :maxdepth: 2 kernel-documentation - media/media_uapi - media/media_kapi - media/dvb-drivers/index - media/v4l-drivers/index + media/index gpu/index Indices and tables diff --git a/Documentation/media/conf.py b/Documentation/media/conf.py new file mode 100644 index 000000000000..62bdba237917 --- /dev/null +++ b/Documentation/media/conf.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8; mode: python -*- + +project = 'Linux Media Subsystem Documentation' diff --git a/Documentation/media/index.rst b/Documentation/media/index.rst new file mode 100644 index 000000000000..e85c557eeea3 --- /dev/null +++ b/Documentation/media/index.rst @@ -0,0 +1,12 @@ +Linux Media Subsystem Documentation +=================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + media_uapi + media_kapi + dvb-drivers/index + v4l-drivers/index From 482941aa84a6224b3e45f8d52de75c0c0de51391 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 13 Aug 2016 16:12:44 +0200 Subject: [PATCH 08/51] doc-rst: add media/conf_nitpick.py The media/conf_nitpick.py is a *build-theme* wich uses the nit-picking mode of sphinx. To compile only the html of 'media' with the nit-picking build use:: make SPHINXDIRS=media SPHINX_CONF=conf_nitpick.py htmldocs With this, the Documentation/conf.py is read first and updated with the configuration values from the Documentation/media/conf_nitpick.py. The origin media/conf_nitpick.py comes from Mauro's experimental docs-next branch:: https://git.linuxtv.org/mchehab/experimental.git mchehab/docs-next BTW fixed python indentation in media/conf_nitpick.py. Python indentation is 4 spaces [1] and Python 3 disallows mixing the use of tabs and spaces for indentation [2]. [1] https://www.python.org/dev/peps/pep-0008/#indentation [2] https://www.python.org/dev/peps/pep-0008/#tabs-or-spaces Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/media/conf_nitpick.py | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 Documentation/media/conf_nitpick.py diff --git a/Documentation/media/conf_nitpick.py b/Documentation/media/conf_nitpick.py new file mode 100644 index 000000000000..11beac2e68fb --- /dev/null +++ b/Documentation/media/conf_nitpick.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8; mode: python -*- + +project = 'Linux Media Subsystem Documentation' + +# It is possible to run Sphinx in nickpick mode with: +nitpicky = True + +# within nit-picking build, do not refer to any intersphinx object +intersphinx_mapping = {} + +# In nickpick mode, it will complain about lots of missing references that +# +# 1) are just typedefs like: bool, __u32, etc; +# 2) It will complain for things like: enum, NULL; +# 3) It will complain for symbols that should be on different +# books (but currently aren't ported to ReST) +# +# The list below has a list of such symbols to be ignored in nitpick mode +# +nitpick_ignore = [ + ("c:func", "clock_gettime"), + ("c:func", "close"), + ("c:func", "container_of"), + ("c:func", "determine_valid_ioctls"), + ("c:func", "ERR_PTR"), + ("c:func", "ioctl"), + ("c:func", "IS_ERR"), + ("c:func", "mmap"), + ("c:func", "open"), + ("c:func", "pci_name"), + ("c:func", "poll"), + ("c:func", "PTR_ERR"), + ("c:func", "read"), + ("c:func", "release"), + ("c:func", "set"), + ("c:func", "struct fd_set"), + ("c:func", "struct pollfd"), + ("c:func", "usb_make_path"), + ("c:func", "write"), + ("c:type", "atomic_t"), + ("c:type", "bool"), + ("c:type", "buf_queue"), + ("c:type", "device"), + ("c:type", "device_driver"), + ("c:type", "device_node"), + ("c:type", "enum"), + ("c:type", "file"), + ("c:type", "i2c_adapter"), + ("c:type", "i2c_board_info"), + ("c:type", "i2c_client"), + ("c:type", "ktime_t"), + ("c:type", "led_classdev_flash"), + ("c:type", "list_head"), + ("c:type", "lock_class_key"), + ("c:type", "module"), + ("c:type", "mutex"), + ("c:type", "pci_dev"), + ("c:type", "pdvbdev"), + ("c:type", "poll_table_struct"), + ("c:type", "s32"), + ("c:type", "s64"), + ("c:type", "sd"), + ("c:type", "spi_board_info"), + ("c:type", "spi_device"), + ("c:type", "spi_master"), + ("c:type", "struct fb_fix_screeninfo"), + ("c:type", "struct pollfd"), + ("c:type", "struct timeval"), + ("c:type", "struct video_capability"), + ("c:type", "u16"), + ("c:type", "u32"), + ("c:type", "u64"), + ("c:type", "u8"), + ("c:type", "union"), + ("c:type", "usb_device"), + + ("cpp:type", "boolean"), + ("cpp:type", "fd"), + ("cpp:type", "fd_set"), + ("cpp:type", "int16_t"), + ("cpp:type", "NULL"), + ("cpp:type", "off_t"), + ("cpp:type", "pollfd"), + ("cpp:type", "size_t"), + ("cpp:type", "ssize_t"), + ("cpp:type", "timeval"), + ("cpp:type", "__u16"), + ("cpp:type", "__u32"), + ("cpp:type", "__u64"), + ("cpp:type", "uint16_t"), + ("cpp:type", "uint32_t"), + ("cpp:type", "video_system_t"), +] From 666756252deede35014e9ffd4bda70127d7dddf1 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 13 Aug 2016 16:12:45 +0200 Subject: [PATCH 09/51] doc-rst: add stand-alone conf.py to gpu folder With the gpu/conf.py, the gpu folder can be build and distributed stand-alone. To compile only the html of 'gpu' folder use:: make SPHINXDIRS="gpu" htmldocs Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/gpu/conf.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Documentation/gpu/conf.py diff --git a/Documentation/gpu/conf.py b/Documentation/gpu/conf.py new file mode 100644 index 000000000000..d60bcd0db040 --- /dev/null +++ b/Documentation/gpu/conf.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8; mode: python -*- + +project = "Linux GPU Driver Developer's Guide" From a55a51da7a8d6e306a025adc1fb21cd2379ca9bd Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Sat, 13 Aug 2016 16:12:46 +0200 Subject: [PATCH 10/51] doc-rst: add docutils config file To stop the sphinx-build on severe errors and exit with an exit code (to stop make) the halt_level must be set. The halt_level can't be set from sphinx, it is a docutils configuration [1]. For this a docutils.conf was added. [1] http://docutils.sourceforge.net/docs/user/config.html Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/docutils.conf | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 Documentation/docutils.conf diff --git a/Documentation/docutils.conf b/Documentation/docutils.conf new file mode 100644 index 000000000000..2830772264c8 --- /dev/null +++ b/Documentation/docutils.conf @@ -0,0 +1,7 @@ +# -*- coding: utf-8 mode: conf-colon -*- +# +# docutils configuration file +# http://docutils.sourceforge.net/docs/user/config.html + +[general] +halt_level: severe \ No newline at end of file From 6203a642785a7979ec64406da0b452037df31e75 Mon Sep 17 00:00:00 2001 From: Andi Shyti Date: Fri, 12 Aug 2016 14:41:25 +0200 Subject: [PATCH 11/51] Documentation: clk: update file names containing referenced structures Commit 'b09d6d991' removes include/linux/clk-private.h and re-arranges the clock related structures contained in it in different files. The documentation has not been updated accordingly, thus it wasn't anymore consistent. Place the structures referenced by Documentation/clk.txt in the correct files and update their contents to the latest status. Signed-off-by: Andi Shyti [geert: Fix path to clk.c, whitespace, more clk_core, ...] Signed-off-by: Geert Uytterhoeven Signed-off-by: Jonathan Corbet --- Documentation/clk.txt | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/Documentation/clk.txt b/Documentation/clk.txt index 5c4bc4d01d0c..22f026aa2f34 100644 --- a/Documentation/clk.txt +++ b/Documentation/clk.txt @@ -31,24 +31,25 @@ serve as a convenient shorthand for the implementation of the hardware-specific bits for the hypothetical "foo" hardware. Tying the two halves of this interface together is struct clk_hw, which -is defined in struct clk_foo and pointed to within struct clk. This +is defined in struct clk_foo and pointed to within struct clk_core. This allows for easy navigation between the two discrete halves of the common clock interface. Part 2 - common data structures and api -Below is the common struct clk definition from -include/linux/clk-private.h, modified for brevity: +Below is the common struct clk_core definition from +drivers/clk/clk.c, modified for brevity: - struct clk { + struct clk_core { const char *name; const struct clk_ops *ops; struct clk_hw *hw; - char **parent_names; - struct clk **parents; - struct clk *parent; - struct hlist_head children; - struct hlist_node child_node; + struct module *owner; + struct clk_core *parent; + const char **parent_names; + struct clk_core **parents; + u8 num_parents; + u8 new_parent_index; ... }; @@ -56,16 +57,19 @@ The members above make up the core of the clk tree topology. The clk api itself defines several driver-facing functions which operate on struct clk. That api is documented in include/linux/clk.h. -Platforms and devices utilizing the common struct clk use the struct -clk_ops pointer in struct clk to perform the hardware-specific parts of -the operations defined in clk.h: +Platforms and devices utilizing the common struct clk_core use the struct +clk_ops pointer in struct clk_core to perform the hardware-specific parts of +the operations defined in clk-provider.h: struct clk_ops { int (*prepare)(struct clk_hw *hw); void (*unprepare)(struct clk_hw *hw); + int (*is_prepared)(struct clk_hw *hw); + void (*unprepare_unused)(struct clk_hw *hw); int (*enable)(struct clk_hw *hw); void (*disable)(struct clk_hw *hw); int (*is_enabled)(struct clk_hw *hw); + void (*disable_unused)(struct clk_hw *hw); unsigned long (*recalc_rate)(struct clk_hw *hw, unsigned long parent_rate); long (*round_rate)(struct clk_hw *hw, @@ -84,6 +88,8 @@ the operations defined in clk.h: u8 index); unsigned long (*recalc_accuracy)(struct clk_hw *hw, unsigned long parent_accuracy); + int (*get_phase)(struct clk_hw *hw); + int (*set_phase)(struct clk_hw *hw, int degrees); void (*init)(struct clk_hw *hw); int (*debug_init)(struct clk_hw *hw, struct dentry *dentry); @@ -91,7 +97,7 @@ the operations defined in clk.h: Part 3 - hardware clk implementations -The strength of the common struct clk comes from its .ops and .hw pointers +The strength of the common struct clk_core comes from its .ops and .hw pointers which abstract the details of struct clk from the hardware-specific bits, and vice versa. To illustrate consider the simple gateable clk implementation in drivers/clk/clk-gate.c: @@ -107,7 +113,7 @@ struct clk_gate contains struct clk_hw hw as well as hardware-specific knowledge about which register and bit controls this clk's gating. Nothing about clock topology or accounting, such as enable_count or notifier_count, is needed here. That is all handled by the common -framework code and struct clk. +framework code and struct clk_core. Let's walk through enabling this clk from driver code: @@ -139,22 +145,18 @@ static void clk_gate_set_bit(struct clk_gate *gate) Note that to_clk_gate is defined as: -#define to_clk_gate(_hw) container_of(_hw, struct clk_gate, clk) +#define to_clk_gate(_hw) container_of(_hw, struct clk_gate, hw) This pattern of abstraction is used for every clock hardware representation. Part 4 - supporting your own clk hardware -When implementing support for a new type of clock it only necessary to +When implementing support for a new type of clock it is only necessary to include the following header: #include -include/linux/clk.h is included within that header and clk-private.h -must never be included from the code which implements the operations for -a clock. More on that below in Part 5. - To construct a clk hardware structure for your platform you must define the following: From d1669c8288a2c86daa6fd59c7fb938c08589d053 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Fri, 5 Aug 2016 11:19:43 +0200 Subject: [PATCH 12/51] doc-rst: customize RTD theme, drop padding of inline literal Remove the distracting (left/right) padding of inline literals. (HTML ). Requested and discussed in [1]. [1] http://www.spinics.net/lists/linux-media/msg103991.html Signed-off-by: Markus Heiser Acked-by: Hans Verkuil Signed-off-by: Jonathan Corbet --- Documentation/sphinx-static/theme_overrides.css | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Documentation/sphinx-static/theme_overrides.css b/Documentation/sphinx-static/theme_overrides.css index 3a2ac4bcfd78..e88461c4c1e6 100644 --- a/Documentation/sphinx-static/theme_overrides.css +++ b/Documentation/sphinx-static/theme_overrides.css @@ -42,11 +42,12 @@ caption a.headerlink { opacity: 0; } caption a.headerlink:hover { opacity: 1; } - /* inline literal: drop the borderbox and red color */ + /* inline literal: drop the borderbox, padding and red color */ code, .rst-content tt, .rst-content code { color: inherit; border: none; + padding: unset; background: inherit; font-size: 85%; } From 35db7e94cdee495673360f449cdae726d5576ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20A=2E=20Holm?= Date: Tue, 26 Jul 2016 15:21:33 +0200 Subject: [PATCH 13/51] README: Delete obsolete i386 info + update arch/i386/ paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Support for i386 was removed in v3.8, delete the paragraph that says processor types above 386 won't work on that architecture. It's obsolete information and potentially confusing. Also change a couple of "arch/i386/" paths to one that exists now, using "arch/x86/" instead. Signed-off-by: Øyvind A. Holm Signed-off-by: Jonathan Corbet --- README | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README b/README index e8c8a6dc1c2b..09f34f78f2bb 100644 --- a/README +++ b/README @@ -229,10 +229,6 @@ CONFIGURING the kernel: under some circumstances lead to problems: probing for a nonexistent controller card may confuse your other controllers - - Compiling the kernel with "Processor type" set higher than 386 - will result in a kernel that does NOT work on a 386. The - kernel will detect this on bootup, and give up. - - A kernel with math-emulation compiled in will still use the coprocessor if one is present: the math emulation will just never get used in that case. The kernel will be slightly larger, @@ -289,7 +285,7 @@ COMPILING the kernel: LOCALVERSION can be set in the "General Setup" menu. - In order to boot your new kernel, you'll need to copy the kernel - image (e.g. .../linux/arch/i386/boot/bzImage after compilation) + image (e.g. .../linux/arch/x86/boot/bzImage after compilation) to the place where your regular bootable kernel is found. - Booting a kernel directly from a floppy without the assistance of a @@ -391,7 +387,7 @@ IF SOMETHING GOES WRONG: - Alternatively, you can use gdb on a running kernel. (read-only; i.e. you cannot change values or set break points.) To do this, first compile the - kernel with -g; edit arch/i386/Makefile appropriately, then do a "make + kernel with -g; edit arch/x86/Makefile appropriately, then do a "make clean". You'll also need to enable CONFIG_PROC_FS (via "make config"). After you've rebooted with the new kernel, do "gdb vmlinux /proc/kcore". From 865a1caa4b6b886babdd9d67e7c3608be4567a51 Mon Sep 17 00:00:00 2001 From: Jean Delvare Date: Mon, 25 Jul 2016 14:29:06 +0200 Subject: [PATCH 14/51] CodingStyle: Clarify and complete chapter 7 Chapter 7 (Centralized exiting of functions) of the coding style documentation is unclear at times, and lacks some information (such as the possibility to indent labels with a single space.) Clarify and complete it. Signed-off-by: Jean Delvare Cc: Markus Elfring Cc: Jonathan Corbet Signed-off-by: Jonathan Corbet --- Documentation/CodingStyle | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Documentation/CodingStyle b/Documentation/CodingStyle index a096836723ca..784226e396fe 100644 --- a/Documentation/CodingStyle +++ b/Documentation/CodingStyle @@ -396,9 +396,13 @@ locations and some common work such as cleanup has to be done. If there is no cleanup needed then just return directly. Choose label names which say what the goto does or why the goto exists. An -example of a good name could be "out_buffer:" if the goto frees "buffer". Avoid -using GW-BASIC names like "err1:" and "err2:". Also don't name them after the -goto location like "err_kmalloc_failed:" +example of a good name could be "out_free_buffer:" if the goto frees "buffer". +Avoid using GW-BASIC names like "err1:" and "err2:", as you would have to +renumber them if you ever add or remove exit paths, and they make correctness +difficult to verify anyway. + +It is advised to indent labels with a single space (not tab), so that +"diff -p" does not confuse labels with functions. The rationale for using gotos is: @@ -425,20 +429,29 @@ The rationale for using gotos is: goto out_buffer; } ... - out_buffer: + out_free_buffer: kfree(buffer); return result; } A common type of bug to be aware of is "one err bugs" which look like this: - err: + err: kfree(foo->bar); kfree(foo); return ret; The bug in this code is that on some exit paths "foo" is NULL. Normally the -fix for this is to split it up into two error labels "err_bar:" and "err_foo:". +fix for this is to split it up into two error labels "err_free_bar:" and +"err_free_foo:": + + err_free_bar: + kfree(foo->bar); + err_free_foo: + kfree(foo); + return ret; + +Ideally you should simulate errors to test all exit paths. Chapter 8: Commenting From a78a136fa9337fdc25fdbaa2d253f9b4dc90ad44 Mon Sep 17 00:00:00 2001 From: Joe Perches Date: Tue, 12 Jul 2016 17:18:28 -0700 Subject: [PATCH 15/51] CodingStyle: Remove "Don't use C99-style comments" Because Linus may still be reading source code on greenbar paper instead of color terminals with code syntax highlighting and appropriate font decorations. Link: http://lkml.kernel.org/r/CA+55aFyQYJerovMsSoSKS7PessZBr4vNp-3QUUwhqk4A4_jcbg@mail.gmail.com Signed-off-by: Joe Perches --- Documentation/CodingStyle | 3 --- 1 file changed, 3 deletions(-) diff --git a/Documentation/CodingStyle b/Documentation/CodingStyle index 784226e396fe..0f1dbd87eb48 100644 --- a/Documentation/CodingStyle +++ b/Documentation/CodingStyle @@ -474,9 +474,6 @@ When commenting the kernel API functions, please use the kernel-doc format. See the files Documentation/kernel-documentation.rst and scripts/kernel-doc for details. -Linux style for comments is the C89 "/* ... */" style. -Don't use C99-style "// ..." comments. - The preferred style for long (multi-line) comments is: /* From 7a9011db32b5a24f29e725220d0ba9e970412f47 Mon Sep 17 00:00:00 2001 From: "David A. Long" Date: Fri, 12 Aug 2016 16:24:44 -0400 Subject: [PATCH 16/51] Documentation: kprobes: Document jprobes stack copying limitations Some architectures (i.e.: sparc64 and arm64) make reasonable partial stack duplication for jprobes problematic. Document this. Signed-off-by: David A. Long Acked-by: Catalin Marinas Acked-by: Masami Hiramatsu Signed-off-by: Jonathan Corbet --- Documentation/kprobes.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Documentation/kprobes.txt b/Documentation/kprobes.txt index 1f9b3e2b98ae..1f6d45abfe42 100644 --- a/Documentation/kprobes.txt +++ b/Documentation/kprobes.txt @@ -103,6 +103,16 @@ Note that the probed function's args may be passed on the stack or in registers. The jprobe will work in either case, so long as the handler's prototype matches that of the probed function. +Note that in some architectures (e.g.: arm64 and sparc64) the stack +copy is not done, as the actual location of stacked parameters may be +outside of a reasonable MAX_STACK_SIZE value and because that location +cannot be determined by the jprobes code. In this case the jprobes +user must be careful to make certain the calling signature of the +function does not cause parameters to be passed on the stack (e.g.: +more than eight function arguments, an argument of more than sixteen +bytes, or more than 64 bytes of argument data, depending on +architecture). + 1.3 Return Probes 1.3.1 How Does a Return Probe Work? From 5021bb933ae299ed494da7350303700077de20fc Mon Sep 17 00:00:00 2001 From: Ricardo Ribalda Delgado Date: Thu, 18 Aug 2016 10:54:19 +0200 Subject: [PATCH 17/51] Documentation: rs485: Do not define manually the ioctl It is not a very good practice to define the IOCTL manually instead of using the header file. Fix it on the documentation example. Signed-off-by: Ricardo Ribalda Delgado Signed-off-by: Jonathan Corbet --- Documentation/serial/serial-rs485.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Documentation/serial/serial-rs485.txt b/Documentation/serial/serial-rs485.txt index 2253b8b45a74..389fcd4759e9 100644 --- a/Documentation/serial/serial-rs485.txt +++ b/Documentation/serial/serial-rs485.txt @@ -45,9 +45,8 @@ #include - /* RS485 ioctls: */ - #define TIOCGRS485 0x542E - #define TIOCSRS485 0x542F + /* Include definition for RS485 ioctls: TIOCGRS485 and TIOCSRS485 */ + #include /* Open your specific device (e.g., /dev/mydevice): */ int fd = open ("/dev/mydevice", O_RDWR); From 00445edff345b654a074733d5a41c79d5fe33be3 Mon Sep 17 00:00:00 2001 From: Icenowy Zheng Date: Sat, 13 Aug 2016 18:01:13 +0800 Subject: [PATCH 18/51] Documentation: sunxi: Update Allwinner SoC documentation Now, the A83T and A64 SoC user manuals are available. Update the documentation to add the links. An updated version of A83T datasheet is also included now. Signed-off-by: Icenowy Zheng Acked-by: Chen-Yu Tsai Signed-off-by: Jonathan Corbet --- Documentation/arm/sunxi/README | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Documentation/arm/sunxi/README b/Documentation/arm/sunxi/README index e5a115f24471..c7a0554523da 100644 --- a/Documentation/arm/sunxi/README +++ b/Documentation/arm/sunxi/README @@ -73,4 +73,13 @@ SunXi family * Octa ARM Cortex-A7 based SoCs - Allwinner A83T + Datasheet - http://dl.linux-sunxi.org/A83T/A83T_datasheet_Revision_1.1.pdf + https://github.com/allwinner-zh/documents/raw/master/A83T/A83T_Datasheet_v1.3_20150510.pdf + + User Manual + https://github.com/allwinner-zh/documents/raw/master/A83T/A83T_User_Manual_v1.5.1_20150513.pdf + + * Quad ARM Cortex-A53 based SoCs + - Allwinner A64 + + Datasheet + http://dl.linux-sunxi.org/A64/A64_Datasheet_V1.1.pdf + + User Manual + http://dl.linux-sunxi.org/A64/Allwinner%20A64%20User%20Manual%20v1.0.pdf From 22c6bbe41ecf7606c3effc92b54d5e939e806866 Mon Sep 17 00:00:00 2001 From: "baolex.ni" Date: Mon, 11 Jul 2016 09:57:37 +0800 Subject: [PATCH 19/51] Update the maximum depth of C-state from 6 to 9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hi Jon, This patch is an old one, we have corrected some minor issues on the newer one. Please only review the newest version from my last mail with this subject "[PATCH] ACPI: Update the maximum depth of C-state from 6 to 9". And I also attached it to this mail. Thanks, Baole On 7/11/2016 6:37 AM, Jonathan Corbet wrote: > On Mon, 4 Jul 2016 09:55:10 +0800 > "baolex.ni" wrote: > >> Currently, CPUIDLE_STATE_MAX has been defined as 10 in the cpuidle head file, >> and max_cstate = CPUIDLE_STATE_MAX – 1, so 9 is the right maximum depth of C-state. >> This change is reflected in one place of the kernel-param file, >> but not in the other place where I suggest changing. >> >> Signed-off-by: Chuansheng Liu >> Signed-off-by: Baole Ni > > So why are there two signoffs on a single-line patch? Which one of you > is the actual author? > > Thanks, > > jon > From cf5f8aa6885874f6490b11507d3c0c86fa0a11f4 Mon Sep 17 00:00:00 2001 From: Chuansheng Liu Date: Mon, 4 Jul 2016 08:52:51 +0800 Subject: [PATCH] Update the maximum depth of C-state from 6 to 9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, CPUIDLE_STATE_MAX has been defined as 10 in the cpuidle head file, and max_cstate = CPUIDLE_STATE_MAX – 1, so 9 is the right maximum depth of C-state. This change is reflected in one place of the kernel-param file, but not in the other place where I suggest changing. Signed-off-by: Chuansheng Liu Signed-off-by: Baole Ni Signed-off-by: Jonathan Corbet --- Documentation/kernel-parameters.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 46c030a49186..20557efa8bce 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1688,7 +1688,7 @@ bytes respectively. Such letter suffixes can also be entirely omitted. intel_idle.max_cstate= [KNL,HW,ACPI,X86] 0 disables intel_idle and fall back on acpi_idle. - 1 to 6 specify maximum depth of C-state. + 1 to 9 specify maximum depth of C-state. intel_pstate= [X86] disable From cfd7c612baa61af61b2c57b7b8321adf2d1b5d90 Mon Sep 17 00:00:00 2001 From: Eugeniu Rosca Date: Wed, 3 Aug 2016 00:40:34 +0200 Subject: [PATCH 20/51] kconfig-language: improve menuconfig usage description Improper menuconfig usage leads to empty menu entries. zconfdump() is able to reveal some real-life examples: - menuconfig VFIO_NOIOMMU - menuconfig RESET_CONTROLLER - menuconfig SND_ARM To avoid future occurrences of those, improve the menuconfig syntax description. Signed-off-by: Eugeniu Rosca Signed-off-by: Jonathan Corbet --- Documentation/kbuild/kconfig-language.txt | 39 ++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/Documentation/kbuild/kconfig-language.txt b/Documentation/kbuild/kconfig-language.txt index db101857b2c9..069fcb3eef6e 100644 --- a/Documentation/kbuild/kconfig-language.txt +++ b/Documentation/kbuild/kconfig-language.txt @@ -274,7 +274,44 @@ menuconfig: This is similar to the simple config entry above, but it also gives a hint to front ends, that all suboptions should be displayed as a -separate list of options. +separate list of options. To make sure all the suboptions will really +show up under the menuconfig entry and not outside of it, every item +from the list must depend on the menuconfig symbol. +In practice, this is achieved by using one of the next two constructs: + +(1): +menuconfig M +if M + config C1 + config C2 +endif + +(2): +menuconfig M +config C1 + depends on M +config C2 + depends on M + +In the following examples (3) and (4), C1 and C2 still have the M +dependency, but will not appear under menuconfig M anymore, because +of C0, which doesn't depend on M: + +(3): +menuconfig M + config C0 +if M + config C1 + config C2 +endif + +(4): +menuconfig M +config C0 +config C1 + depends on M +config C2 + depends on M choices: From 505f711174b03f42bdfae1aa6cc191b26c157e94 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 15 Aug 2016 17:15:59 +0200 Subject: [PATCH 21/51] doc-rst: add index to sub-folders Add a index if only a sub-folder is build e.g.:: make SPHINXDIRS=media cleandocs htmldocs BTW: removed dead search link in the top-index file Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/gpu/conf.py | 2 ++ Documentation/gpu/index.rst | 7 +++++++ Documentation/index.rst | 1 - Documentation/media/conf.py | 2 ++ Documentation/media/index.rst | 7 +++++++ 5 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Documentation/gpu/conf.py b/Documentation/gpu/conf.py index d60bcd0db040..6314d1708230 100644 --- a/Documentation/gpu/conf.py +++ b/Documentation/gpu/conf.py @@ -1,3 +1,5 @@ # -*- coding: utf-8; mode: python -*- project = "Linux GPU Driver Developer's Guide" + +tags.add("subproject") diff --git a/Documentation/gpu/index.rst b/Documentation/gpu/index.rst index fcac0fa72056..5ff3d2b236af 100644 --- a/Documentation/gpu/index.rst +++ b/Documentation/gpu/index.rst @@ -12,3 +12,10 @@ Linux GPU Driver Developer's Guide drm-uapi i915 vga-switcheroo + +.. only:: subproject + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/index.rst b/Documentation/index.rst index bdd9525e05aa..a15f81855b39 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -19,4 +19,3 @@ Indices and tables ================== * :ref:`genindex` -* :ref:`search` diff --git a/Documentation/media/conf.py b/Documentation/media/conf.py index 62bdba237917..77cb2bbd9461 100644 --- a/Documentation/media/conf.py +++ b/Documentation/media/conf.py @@ -1,3 +1,5 @@ # -*- coding: utf-8; mode: python -*- project = 'Linux Media Subsystem Documentation' + +tags.add("subproject") diff --git a/Documentation/media/index.rst b/Documentation/media/index.rst index e85c557eeea3..7f8f0af620ce 100644 --- a/Documentation/media/index.rst +++ b/Documentation/media/index.rst @@ -10,3 +10,10 @@ Contents: media_kapi dvb-drivers/index v4l-drivers/index + +.. only:: subproject + + Indices + ======= + + * :ref:`genindex` From 59058a7adc4b10f357cc5e7cbcda0b2d45940258 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Tue, 16 Aug 2016 09:30:31 -0700 Subject: [PATCH 22/51] mm, kasan: Update kasan docs to indicate arm64 support KASAN has been supported on arm64 since 39d114ddc682 ("arm64: add KASAN support"). Update the docs to indicate this. Signed-off-by: Laura Abbott Acked-by: Andrey Ryabinin Signed-off-by: Jonathan Corbet --- Documentation/kasan.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/kasan.txt b/Documentation/kasan.txt index 7dd95b35cd7c..d167220324c4 100644 --- a/Documentation/kasan.txt +++ b/Documentation/kasan.txt @@ -12,7 +12,7 @@ KASAN uses compile-time instrumentation for checking every memory access, therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is required for detection of out-of-bounds accesses to stack or global variables. -Currently KASAN is supported only for x86_64 architecture. +Currently KASAN is supported only for x86_64 and arm64 architecture. 1. Usage ======== From 16dbe8042eb5df72ee761c24a383e4a5e11d540b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:35 -0300 Subject: [PATCH 23/51] docs-rst: fix a breakage when building PDF documents changeset 606b9ac81a63 ("doc-rst: generic way to build only sphinx sub-folders") accidentally broke PDF generation by adding an extra ")". Remove it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/Makefile.sphinx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index ea0664cece12..fdef3a4bc8c7 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -71,7 +71,7 @@ ifeq ($(HAVE_PDFLATEX),0) $(warning The 'pdflatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) @echo " SKIP Sphinx $@ target." else # HAVE_PDFLATEX - @$(call loop_cmd,sphinx,latex,.,latex,.)) + @$(call loop_cmd,sphinx,latex,.,latex,.) $(Q)$(MAKE) -C $(BUILDDIR)/latex endif # HAVE_PDFLATEX From a69ab10869941f45b5f1291ec482df1e9bd7895f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:36 -0300 Subject: [PATCH 24/51] docs-rst: remove a rst2pdf left over code The usage of rst2pdf was replaced by pdflatex on a previous patch. Remove the left-over code at conf.py. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 43e41ed74584..1c6d9da5d1b5 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -31,13 +31,6 @@ from load_config import loadConfig # ones. extensions = ['kernel-doc', 'rstFlatTable', 'kernel_include'] -# Gracefully handle missing rst2pdf. -try: - import rst2pdf - extensions += ['rst2pdf.pdfbuilder'] -except ImportError: - pass - # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From caee5cdede9c9b3562abd6a93c71e791577802ad Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:37 -0300 Subject: [PATCH 25/51] docs-rst: allow generating some LaTeX pages in landscape Portrait is too small for some tables used at the media docs. So, allow documents to tell Sphinx to generate some pages in landscape by using: .. raw:: latex \begin{landscape} .. raw:: latex \end{landscape} Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 1c6d9da5d1b5..38b240073336 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -246,16 +246,19 @@ htmlhelp_basename = 'TheLinuxKerneldoc' latex_elements = { # The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', +'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', +'pointsize': '10pt', # Latex figure (float) alignment #'figure_align': 'htbp', + +# Additional stuff for the LaTeX preamble. + 'preamble': ''' + % Allow generate some pages in landscape + \\usepackage{lscape} + ''' } # Grouping the document tree into LaTeX files. List of tuples From 60459774659267569b4d944fcecb372ad0c48629 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:38 -0300 Subject: [PATCH 26/51] docs-rst: improve output for .. notes:: on LaTeX The output for those notes are bad in pdf, as they're not in a box with a different color. Also, it causes the output to not build if the note is inside a table. Change its implementation to avoid the above troubles. The logic there came from: https://stackoverflow.com/questions/606746/how-to-customize-an-existing-latex-environment-without-interfering-with-other-en Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/Documentation/conf.py b/Documentation/conf.py index 38b240073336..429183a03b92 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -258,6 +258,29 @@ latex_elements = { 'preamble': ''' % Allow generate some pages in landscape \\usepackage{lscape} + + % Put notes in gray color and let them be inside a table + + \\definecolor{MyGray}{rgb}{0.80,0.80,0.80} + + \\makeatletter\\newenvironment{graybox}{% + \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\columnwidth}}{\\end{minipage}\\end{lrbox}% + \\colorbox{MyGray}{\\usebox{\\@tempboxa}} + }\\makeatother + + \\makeatletter + \\renewenvironment{notice}[2]{ + \\begin{graybox} + \\bf\\it + \\def\\py@noticetype{#1} + \\par\\strong{#2} + \\csname py@noticestart@#1\\endcsname + } + { + \\csname py@noticeend@\\py@noticetype\\endcsname + \\end{graybox} + } + \\makeatother ''' } From a682ec4ba10c88231cdbb8bb9823b2cc749d6364 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:39 -0300 Subject: [PATCH 27/51] docs-rst: Don't mangle with UTF-8 chars on LaTeX/PDF output pdflatex doesn't accept using some UTF-8 chars, like "equal or less than" or "equal or greater than" chars. However, the media documents use them. So, we need to use XeLaTeX for conversion, and a font that accepts such characters. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/Makefile.sphinx | 6 +++--- Documentation/conf.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index fdef3a4bc8c7..16a3502c9e40 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -29,7 +29,7 @@ else ifneq ($(DOCBOOKS),) else # HAVE_SPHINX # User-friendly check for pdflatex -HAVE_PDFLATEX := $(shell if which pdflatex >/dev/null 2>&1; then echo 1; else echo 0; fi) +HAVE_PDFLATEX := $(shell if which xelatex >/dev/null 2>&1; then echo 1; else echo 0; fi) # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 @@ -68,11 +68,11 @@ htmldocs: pdfdocs: ifeq ($(HAVE_PDFLATEX),0) - $(warning The 'pdflatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) + $(warning The 'xelatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) @echo " SKIP Sphinx $@ target." else # HAVE_PDFLATEX @$(call loop_cmd,sphinx,latex,.,latex,.) - $(Q)$(MAKE) -C $(BUILDDIR)/latex + $(Q)$(MAKE) PDFLATEX=xelatex -C $(BUILDDIR)/latex endif # HAVE_PDFLATEX epubdocs: diff --git a/Documentation/conf.py b/Documentation/conf.py index 429183a03b92..06e6db916d2f 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -254,6 +254,10 @@ latex_elements = { # Latex figure (float) alignment #'figure_align': 'htbp', +# Don't mangle with UTF-8 chars +'inputenc': '', +'utf8extra': '', + # Additional stuff for the LaTeX preamble. 'preamble': ''' % Allow generate some pages in landscape @@ -281,6 +285,13 @@ latex_elements = { \\end{graybox} } \\makeatother + + % Use some font with UTF-8 support with XeLaTeX + \\usepackage{fontspec} + \\setsansfont{DejaVu Serif} + \\setromanfont{DejaVu Sans} + \\setmonofont{DejaVu Sans Mono} + ''' } From 85c21e5c3ee74fb75d690c57f7066bae7e2dca55 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:40 -0300 Subject: [PATCH 28/51] docs-rst: better adjust margins and font size As we have big tables, reduce the left/right margins and decrease the point size to 8pt. Visually, it is still good enough, and now less tables are too big to be displayed. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 06e6db916d2f..61b71ae60ad0 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -249,7 +249,7 @@ latex_elements = { 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). -'pointsize': '10pt', +'pointsize': '8pt', # Latex figure (float) alignment #'figure_align': 'htbp', @@ -260,6 +260,9 @@ latex_elements = { # Additional stuff for the LaTeX preamble. 'preamble': ''' + % Adjust margins + \\usepackage[margin=0.5in, top=1in, bottom=1in]{geometry} + % Allow generate some pages in landscape \\usepackage{lscape} From 999d998eee3dc1a4942b56f178be363844ea3729 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:41 -0300 Subject: [PATCH 29/51] docs-rst: parse-heraders.pl: escape LaTeX characters Let's escape the LaTeX characters, to avoid troubles when outputing them. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/sphinx/parse-headers.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/sphinx/parse-headers.pl b/Documentation/sphinx/parse-headers.pl index 34bd9e2630b0..74089b0da798 100755 --- a/Documentation/sphinx/parse-headers.pl +++ b/Documentation/sphinx/parse-headers.pl @@ -220,7 +220,7 @@ $data =~ s/\n\s+\n/\n\n/g; # # Add escape codes for special characters # -$data =~ s,([\_\`\*\<\>\&\\\\:\/\|]),\\$1,g; +$data =~ s,([\_\`\*\<\>\&\\\\:\/\|\%\$\#\{\}\~\^]),\\$1,g; $data =~ s,DEPRECATED,**DEPRECATED**,g; From 8b8bbf8fe5cfd9aada5f29dc6f6e5b8e4320d058 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:42 -0300 Subject: [PATCH 30/51] docs-rst: Don't go to interactive mode on errors When building for LaTeX, it stops and enters into interactive mode on errors. Don't do that, as there are some non-fatal errors on media books when using Sphinx 1.4.x that we don't know how fix yet. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/Makefile.sphinx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index 16a3502c9e40..ba4efb1f68f3 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -72,7 +72,7 @@ ifeq ($(HAVE_PDFLATEX),0) @echo " SKIP Sphinx $@ target." else # HAVE_PDFLATEX @$(call loop_cmd,sphinx,latex,.,latex,.) - $(Q)$(MAKE) PDFLATEX=xelatex -C $(BUILDDIR)/latex + $(Q)$(MAKE) PDFLATEX=xelatex LATEXOPTS="-interaction=nonstopmode" -C $(BUILDDIR)/latex endif # HAVE_PDFLATEX epubdocs: From d4fe7e14e4ff5d90286397180831a35df502969e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 16 Aug 2016 13:25:43 -0300 Subject: [PATCH 31/51] docs-rst: enable the Sphinx math extension This extension will be used by the media books. The name of the math image extension changed on Sphinx 1.4.x, according with: http://www.sphinx-doc.org/en/stable/ext/math.html#module-sphinx.ext.imgmath Let's autodetect, to keep building with versions < 1.4. Suggested-by: Markus Heiser Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Documentation/conf.py b/Documentation/conf.py index 61b71ae60ad0..23e2f0bbcfc8 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -14,6 +14,11 @@ import sys import os +import sphinx + +# Get Sphinx version +major, minor, patch = map(int, sphinx.__version__.split(".")) + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -31,6 +36,12 @@ from load_config import loadConfig # ones. extensions = ['kernel-doc', 'rstFlatTable', 'kernel_include'] +# The name of the math extension changed on Sphinx 1.4 +if minor > 3: + extensions.append("sphinx.ext.imgmath") +else: + extensions.append("sphinx.ext.pngmath") + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] From 059c5e918f6613d7062f27705488e5bd72cd09bd Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Aug 2016 16:00:25 -0600 Subject: [PATCH 32/51] docs: create a new dev-tools directory This directory will be a collecting point for documentation oriented around development tools. As a step toward ordering Documentation/ it's a small one, but we have to start somewhere... Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/tools.rst | 16 ++++++++++++++++ Documentation/index.rst | 1 + 2 files changed, 17 insertions(+) create mode 100644 Documentation/dev-tools/tools.rst diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst new file mode 100644 index 000000000000..60ddb9ea6ef9 --- /dev/null +++ b/Documentation/dev-tools/tools.rst @@ -0,0 +1,16 @@ +================================ +Development tools for the kernel +================================ + +This document is a collection of documents about development tools that can +be used to work on the kernel. For now, the documents have been pulled +together without any significant effot to integrate them into a coherent +whole; patches welcome! + +.. class:: toc-title + + Table of contents + +.. toctree:: + :maxdepth: 2 + diff --git a/Documentation/index.rst b/Documentation/index.rst index e0fc72963e87..643fb3205540 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -14,6 +14,7 @@ Contents: :maxdepth: 2 kernel-documentation + dev-tools/tools media/media_uapi media/media_kapi media/dvb-drivers/index From 4b9033a33494ec9154d63e706e9e47f7eb3fd59e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Aug 2016 16:03:14 -0600 Subject: [PATCH 33/51] docs: sphinxify coccinelle.txt and add it to dev-tools No textual changes have been made, but the formatting has obviously been tweaked. Cc: Michal Marek Cc: Gilles Muller Acked-by: Nicolas Palix Acked-by: Julia Lawall Signed-off-by: Jonathan Corbet --- .../coccinelle.rst} | 359 +++++++++--------- Documentation/dev-tools/tools.rst | 1 + MAINTAINERS | 2 +- 3 files changed, 192 insertions(+), 170 deletions(-) rename Documentation/{coccinelle.txt => dev-tools/coccinelle.rst} (56%) diff --git a/Documentation/coccinelle.txt b/Documentation/dev-tools/coccinelle.rst similarity index 56% rename from Documentation/coccinelle.txt rename to Documentation/dev-tools/coccinelle.rst index 01fb1dae3163..4a64b4c69d3f 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/dev-tools/coccinelle.rst @@ -1,10 +1,18 @@ -Copyright 2010 Nicolas Palix -Copyright 2010 Julia Lawall -Copyright 2010 Gilles Muller +.. Copyright 2010 Nicolas Palix +.. Copyright 2010 Julia Lawall +.. Copyright 2010 Gilles Muller +.. highlight:: none - Getting Coccinelle -~~~~~~~~~~~~~~~~~~~~ +Coccinelle +========== + +Coccinelle is a tool for pattern matching and text transformation that has +many uses in kernel development, including the application of complex, +tree-wide patches and detection of problematic programming patterns. + +Getting Coccinelle +------------------- The semantic patches included in the kernel use features and options which are provided by Coccinelle version 1.0.0-rc11 and above. @@ -22,24 +30,23 @@ of many distributions, e.g. : - NetBSD - FreeBSD - You can get the latest version released from the Coccinelle homepage at http://coccinelle.lip6.fr/ Information and tips about Coccinelle are also provided on the wiki pages at http://cocci.ekstranet.diku.dk/wiki/doku.php -Once you have it, run the following command: +Once you have it, run the following command:: ./configure make -as a regular user, and install it with +as a regular user, and install it with:: sudo make install - Supplemental documentation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Supplemental documentation +--------------------------- For supplemental documentation refer to the wiki: @@ -47,49 +54,52 @@ https://bottest.wiki.kernel.org/coccicheck The wiki documentation always refers to the linux-next version of the script. - Using Coccinelle on the Linux kernel -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using Coccinelle on the Linux kernel +------------------------------------ A Coccinelle-specific target is defined in the top level -Makefile. This target is named 'coccicheck' and calls the 'coccicheck' -front-end in the 'scripts' directory. +Makefile. This target is named ``coccicheck`` and calls the ``coccicheck`` +front-end in the ``scripts`` directory. -Four basic modes are defined: patch, report, context, and org. The mode to -use is specified by setting the MODE variable with 'MODE='. +Four basic modes are defined: ``patch``, ``report``, ``context``, and +``org``. The mode to use is specified by setting the MODE variable with +``MODE=``. -'patch' proposes a fix, when possible. +- ``patch`` proposes a fix, when possible. -'report' generates a list in the following format: +- ``report`` generates a list in the following format: file:line:column-column: message -'context' highlights lines of interest and their context in a -diff-like style.Lines of interest are indicated with '-'. +- ``context`` highlights lines of interest and their context in a + diff-like style.Lines of interest are indicated with ``-``. -'org' generates a report in the Org mode format of Emacs. +- ``org`` generates a report in the Org mode format of Emacs. Note that not all semantic patches implement all modes. For easy use of Coccinelle, the default mode is "report". Two other modes provide some common combinations of these modes. -'chain' tries the previous modes in the order above until one succeeds. +- ``chain`` tries the previous modes in the order above until one succeeds. -'rep+ctxt' runs successively the report mode and the context mode. - It should be used with the C option (described later) - which checks the code on a file basis. +- ``rep+ctxt`` runs successively the report mode and the context mode. + It should be used with the C option (described later) + which checks the code on a file basis. -Examples: - To make a report for every semantic patch, run the following command: +Examples +~~~~~~~~ + +To make a report for every semantic patch, run the following command:: make coccicheck MODE=report - To produce patches, run: +To produce patches, run:: make coccicheck MODE=patch The coccicheck target applies every semantic patch available in the -sub-directories of 'scripts/coccinelle' to the entire Linux kernel. +sub-directories of ``scripts/coccinelle`` to the entire Linux kernel. For each semantic patch, a commit message is proposed. It gives a description of the problem being checked by the semantic patch, and @@ -99,15 +109,15 @@ As any static code analyzer, Coccinelle produces false positives. Thus, reports must be carefully checked, and patches reviewed. -To enable verbose messages set the V= variable, for example: +To enable verbose messages set the V= variable, for example:: make coccicheck MODE=report V=1 - Coccinelle parallelization -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Coccinelle parallelization +--------------------------- By default, coccicheck tries to run as parallel as possible. To change -the parallelism, set the J= variable. For example, to run across 4 CPUs: +the parallelism, set the J= variable. For example, to run across 4 CPUs:: make coccicheck MODE=report J=4 @@ -115,44 +125,47 @@ As of Coccinelle 1.0.2 Coccinelle uses Ocaml parmap for parallelization, if support for this is detected you will benefit from parmap parallelization. When parmap is enabled coccicheck will enable dynamic load balancing by using -'--chunksize 1' argument, this ensures we keep feeding threads with work +``--chunksize 1`` argument, this ensures we keep feeding threads with work one by one, so that we avoid the situation where most work gets done by only a few threads. With dynamic load balancing, if a thread finishes early we keep feeding it more work. When parmap is enabled, if an error occurs in Coccinelle, this error -value is propagated back, the return value of the 'make coccicheck' +value is propagated back, the return value of the ``make coccicheck`` captures this return value. - Using Coccinelle with a single semantic patch -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using Coccinelle with a single semantic patch +--------------------------------------------- The optional make variable COCCI can be used to check a single semantic patch. In that case, the variable must be initialized with the name of the semantic patch to apply. -For instance: +For instance:: make coccicheck COCCI= MODE=patch -or + +or:: + make coccicheck COCCI= MODE=report - Controlling Which Files are Processed by Coccinelle -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Controlling Which Files are Processed by Coccinelle +--------------------------------------------------- + By default the entire kernel source tree is checked. -To apply Coccinelle to a specific directory, M= can be used. -For example, to check drivers/net/wireless/ one may write: +To apply Coccinelle to a specific directory, ``M=`` can be used. +For example, to check drivers/net/wireless/ one may write:: make coccicheck M=drivers/net/wireless/ To apply Coccinelle on a file basis, instead of a directory basis, the -following command may be used: +following command may be used:: make C=1 CHECK="scripts/coccicheck" -To check only newly edited code, use the value 2 for the C flag, i.e. +To check only newly edited code, use the value 2 for the C flag, i.e.:: make C=2 CHECK="scripts/coccicheck" @@ -166,8 +179,8 @@ semantic patch as shown in the previous section. The "report" mode is the default. You can select another one with the MODE variable explained above. - Debugging Coccinelle SmPL patches -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Debugging Coccinelle SmPL patches +--------------------------------- Using coccicheck is best as it provides in the spatch command line include options matching the options used when we compile the kernel. @@ -177,8 +190,8 @@ manually run Coccinelle with debug options added. Alternatively you can debug running Coccinelle against SmPL patches by asking for stderr to be redirected to stderr, by default stderr is redirected to /dev/null, if you'd like to capture stderr you -can specify the DEBUG_FILE="file.txt" option to coccicheck. For -instance: +can specify the ``DEBUG_FILE="file.txt"`` option to coccicheck. For +instance:: rm -f cocci.err make coccicheck COCCI=scripts/coccinelle/free/kfree.cocci MODE=report DEBUG_FILE=cocci.err @@ -186,7 +199,7 @@ instance: You can use SPFLAGS to add debugging flags, for instance you may want to add both --profile --show-trying to SPFLAGS when debugging. For instance -you may want to use: +you may want to use:: rm -f err.log export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci @@ -198,24 +211,24 @@ work. DEBUG_FILE support is only supported when using coccinelle >= 1.2. - .cocciconfig support -~~~~~~~~~~~~~~~~~~~~~~ +.cocciconfig support +-------------------- Coccinelle supports reading .cocciconfig for default Coccinelle options that should be used every time spatch is spawned, the order of precedence for variables for .cocciconfig is as follows: - o Your current user's home directory is processed first - o Your directory from which spatch is called is processed next - o The directory provided with the --dir option is processed last, if used +- Your current user's home directory is processed first +- Your directory from which spatch is called is processed next +- The directory provided with the --dir option is processed last, if used Since coccicheck runs through make, it naturally runs from the kernel proper dir, as such the second rule above would be implied for picking up a -.cocciconfig when using 'make coccicheck'. +.cocciconfig when using ``make coccicheck``. -'make coccicheck' also supports using M= targets.If you do not supply +``make coccicheck`` also supports using M= targets.If you do not supply any M= target, it is assumed you want to target the entire kernel. -The kernel coccicheck script has: +The kernel coccicheck script has:: if [ "$KBUILD_EXTMOD" = "" ] ; then OPTIONS="--dir $srctree $COCCIINCLUDE" @@ -235,12 +248,12 @@ override any of the kernel's .coccicheck's settings using SPFLAGS. We help Coccinelle when used against Linux with a set of sensible defaults options for Linux with our own Linux .cocciconfig. This hints to coccinelle -git can be used for 'git grep' queries over coccigrep. A timeout of 200 +git can be used for ``git grep`` queries over coccigrep. A timeout of 200 seconds should suffice for now. The options picked up by coccinelle when reading a .cocciconfig do not appear as arguments to spatch processes running on your system, to confirm what -options will be used by Coccinelle run: +options will be used by Coccinelle run:: spatch --print-options-only @@ -252,219 +265,227 @@ carries its own .cocciconfig, you will need to use SPFLAGS to use idutils if desired. See below section "Additional flags" for more details on how to use idutils. - Additional flags -~~~~~~~~~~~~~~~~~~ +Additional flags +---------------- Additional flags can be passed to spatch through the SPFLAGS variable. This works as Coccinelle respects the last flags -given to it when options are in conflict. +given to it when options are in conflict. :: make SPFLAGS=--use-glimpse coccicheck Coccinelle supports idutils as well but requires coccinelle >= 1.0.6. When no ID file is specified coccinelle assumes your ID database file is in the file .id-utils.index on the top level of the kernel, coccinelle -carries a script scripts/idutils_index.sh which creates the database with +carries a script scripts/idutils_index.sh which creates the database with:: mkid -i C --output .id-utils.index If you have another database filename you can also just symlink with this -name. +name. :: make SPFLAGS=--use-idutils coccicheck Alternatively you can specify the database filename explicitly, for -instance: +instance:: make SPFLAGS="--use-idutils /full-path/to/ID" coccicheck -See spatch --help to learn more about spatch options. +See ``spatch --help`` to learn more about spatch options. -Note that the '--use-glimpse' and '--use-idutils' options +Note that the ``--use-glimpse`` and ``--use-idutils`` options require external tools for indexing the code. None of them is thus active by default. However, by indexing the code with one of these tools, and according to the cocci file used, spatch could proceed the entire code base more quickly. - SmPL patch specific options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SmPL patch specific options +--------------------------- SmPL patches can have their own requirements for options passed to Coccinelle. SmPL patch specific options can be provided by -providing them at the top of the SmPL patch, for instance: +providing them at the top of the SmPL patch, for instance:: -// Options: --no-includes --include-headers + // Options: --no-includes --include-headers - SmPL patch Coccinelle requirements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +SmPL patch Coccinelle requirements +---------------------------------- As Coccinelle features get added some more advanced SmPL patches may require newer versions of Coccinelle. If an SmPL patch requires at least a version of Coccinelle, this can be specified as follows, -as an example if requiring at least Coccinelle >= 1.0.5: +as an example if requiring at least Coccinelle >= 1.0.5:: -// Requires: 1.0.5 + // Requires: 1.0.5 - Proposing new semantic patches -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Proposing new semantic patches +------------------------------- New semantic patches can be proposed and submitted by kernel developers. For sake of clarity, they should be organized in the -sub-directories of 'scripts/coccinelle/'. +sub-directories of ``scripts/coccinelle/``. - Detailed description of the 'report' mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Detailed description of the ``report`` mode +------------------------------------------- + +``report`` generates a list in the following format:: -'report' generates a list in the following format: file:line:column-column: message -Example: +Example +~~~~~~~ -Running +Running:: make coccicheck MODE=report COCCI=scripts/coccinelle/api/err_cast.cocci -will execute the following part of the SmPL script. +will execute the following part of the SmPL script:: - -@r depends on !context && !patch && (org || report)@ -expression x; -position p; -@@ + + @r depends on !context && !patch && (org || report)@ + expression x; + position p; + @@ - ERR_PTR@p(PTR_ERR(x)) + ERR_PTR@p(PTR_ERR(x)) -@script:python depends on report@ -p << r.p; -x << r.x; -@@ + @script:python depends on report@ + p << r.p; + x << r.x; + @@ -msg="ERR_CAST can be used with %s" % (x) -coccilib.report.print_report(p[0], msg) - + msg="ERR_CAST can be used with %s" % (x) + coccilib.report.print_report(p[0], msg) + This SmPL excerpt generates entries on the standard output, as -illustrated below: +illustrated below:: -/home/user/linux/crypto/ctr.c:188:9-16: ERR_CAST can be used with alg -/home/user/linux/crypto/authenc.c:619:9-16: ERR_CAST can be used with auth -/home/user/linux/crypto/xts.c:227:9-16: ERR_CAST can be used with alg + /home/user/linux/crypto/ctr.c:188:9-16: ERR_CAST can be used with alg + /home/user/linux/crypto/authenc.c:619:9-16: ERR_CAST can be used with auth + /home/user/linux/crypto/xts.c:227:9-16: ERR_CAST can be used with alg - Detailed description of the 'patch' mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Detailed description of the ``patch`` mode +------------------------------------------ -When the 'patch' mode is available, it proposes a fix for each problem +When the ``patch`` mode is available, it proposes a fix for each problem identified. -Example: +Example +~~~~~~~ + +Running:: -Running make coccicheck MODE=patch COCCI=scripts/coccinelle/api/err_cast.cocci -will execute the following part of the SmPL script. +will execute the following part of the SmPL script:: - -@ depends on !context && patch && !org && !report @ -expression x; -@@ + + @ depends on !context && patch && !org && !report @ + expression x; + @@ -- ERR_PTR(PTR_ERR(x)) -+ ERR_CAST(x) - + - ERR_PTR(PTR_ERR(x)) + + ERR_CAST(x) + This SmPL excerpt generates patch hunks on the standard output, as -illustrated below: +illustrated below:: -diff -u -p a/crypto/ctr.c b/crypto/ctr.c ---- a/crypto/ctr.c 2010-05-26 10:49:38.000000000 +0200 -+++ b/crypto/ctr.c 2010-06-03 23:44:49.000000000 +0200 -@@ -185,7 +185,7 @@ static struct crypto_instance *crypto_ct + diff -u -p a/crypto/ctr.c b/crypto/ctr.c + --- a/crypto/ctr.c 2010-05-26 10:49:38.000000000 +0200 + +++ b/crypto/ctr.c 2010-06-03 23:44:49.000000000 +0200 + @@ -185,7 +185,7 @@ static struct crypto_instance *crypto_ct alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) -- return ERR_PTR(PTR_ERR(alg)); -+ return ERR_CAST(alg); - + - return ERR_PTR(PTR_ERR(alg)); + + return ERR_CAST(alg); + /* Block size must be >= 4 bytes. */ err = -EINVAL; - Detailed description of the 'context' mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Detailed description of the ``context`` mode +-------------------------------------------- -'context' highlights lines of interest and their context +``context`` highlights lines of interest and their context in a diff-like style. -NOTE: The diff-like output generated is NOT an applicable patch. The - intent of the 'context' mode is to highlight the important lines - (annotated with minus, '-') and gives some surrounding context + **NOTE**: The diff-like output generated is NOT an applicable patch. The + intent of the ``context`` mode is to highlight the important lines + (annotated with minus, ``-``) and gives some surrounding context lines around. This output can be used with the diff mode of Emacs to review the code. -Example: +Example +~~~~~~~ + +Running:: -Running make coccicheck MODE=context COCCI=scripts/coccinelle/api/err_cast.cocci -will execute the following part of the SmPL script. +will execute the following part of the SmPL script:: - -@ depends on context && !patch && !org && !report@ -expression x; -@@ + + @ depends on context && !patch && !org && !report@ + expression x; + @@ -* ERR_PTR(PTR_ERR(x)) - + * ERR_PTR(PTR_ERR(x)) + This SmPL excerpt generates diff hunks on the standard output, as -illustrated below: +illustrated below:: -diff -u -p /home/user/linux/crypto/ctr.c /tmp/nothing ---- /home/user/linux/crypto/ctr.c 2010-05-26 10:49:38.000000000 +0200 -+++ /tmp/nothing -@@ -185,7 +185,6 @@ static struct crypto_instance *crypto_ct + diff -u -p /home/user/linux/crypto/ctr.c /tmp/nothing + --- /home/user/linux/crypto/ctr.c 2010-05-26 10:49:38.000000000 +0200 + +++ /tmp/nothing + @@ -185,7 +185,6 @@ static struct crypto_instance *crypto_ct alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) -- return ERR_PTR(PTR_ERR(alg)); - + - return ERR_PTR(PTR_ERR(alg)); + /* Block size must be >= 4 bytes. */ err = -EINVAL; - Detailed description of the 'org' mode -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Detailed description of the ``org`` mode +---------------------------------------- -'org' generates a report in the Org mode format of Emacs. +``org`` generates a report in the Org mode format of Emacs. -Example: +Example +~~~~~~~ + +Running:: -Running make coccicheck MODE=org COCCI=scripts/coccinelle/api/err_cast.cocci -will execute the following part of the SmPL script. +will execute the following part of the SmPL script:: - -@r depends on !context && !patch && (org || report)@ -expression x; -position p; -@@ + + @r depends on !context && !patch && (org || report)@ + expression x; + position p; + @@ - ERR_PTR@p(PTR_ERR(x)) + ERR_PTR@p(PTR_ERR(x)) -@script:python depends on org@ -p << r.p; -x << r.x; -@@ + @script:python depends on org@ + p << r.p; + x << r.x; + @@ -msg="ERR_CAST can be used with %s" % (x) -msg_safe=msg.replace("[","@(").replace("]",")") -coccilib.org.print_todo(p[0], msg_safe) - + msg="ERR_CAST can be used with %s" % (x) + msg_safe=msg.replace("[","@(").replace("]",")") + coccilib.org.print_todo(p[0], msg_safe) + This SmPL excerpt generates Org entries on the standard output, as -illustrated below: +illustrated below:: -* TODO [[view:/home/user/linux/crypto/ctr.c::face=ovl-face1::linb=188::colb=9::cole=16][ERR_CAST can be used with alg]] -* TODO [[view:/home/user/linux/crypto/authenc.c::face=ovl-face1::linb=619::colb=9::cole=16][ERR_CAST can be used with auth]] -* TODO [[view:/home/user/linux/crypto/xts.c::face=ovl-face1::linb=227::colb=9::cole=16][ERR_CAST can be used with alg]] + * TODO [[view:/home/user/linux/crypto/ctr.c::face=ovl-face1::linb=188::colb=9::cole=16][ERR_CAST can be used with alg]] + * TODO [[view:/home/user/linux/crypto/authenc.c::face=ovl-face1::linb=619::colb=9::cole=16][ERR_CAST can be used with auth]] + * TODO [[view:/home/user/linux/crypto/xts.c::face=ovl-face1::linb=227::colb=9::cole=16][ERR_CAST can be used with alg]] diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 60ddb9ea6ef9..ae0c58c784db 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -14,3 +14,4 @@ whole; patches welcome! .. toctree:: :maxdepth: 2 + coccinelle diff --git a/MAINTAINERS b/MAINTAINERS index 20bb1d00098c..1e5460cfc196 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3124,7 +3124,7 @@ L: cocci@systeme.lip6.fr (moderated for non-subscribers) T: git git://git.kernel.org/pub/scm/linux/kernel/git/mmarek/kbuild.git misc W: http://coccinelle.lip6.fr/ S: Supported -F: Documentation/coccinelle.txt +F: Documentation/dev-tools/coccinelle.rst F: scripts/coccinelle/ F: scripts/coccicheck From d228af5bcb60fda50f8b3a100c0539c4994df040 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:09:14 -0600 Subject: [PATCH 34/51] docs: sphinxify sparse.txt and move to dev-tools Fold the sparse document into the development tools set; no changes to the text itself beyond formatting. Signed-off-by: Jonathan Corbet --- .../{sparse.txt => dev-tools/sparse.rst} | 39 ++++++++++++------- Documentation/dev-tools/tools.rst | 1 + 2 files changed, 25 insertions(+), 15 deletions(-) rename Documentation/{sparse.txt => dev-tools/sparse.rst} (82%) diff --git a/Documentation/sparse.txt b/Documentation/dev-tools/sparse.rst similarity index 82% rename from Documentation/sparse.txt rename to Documentation/dev-tools/sparse.rst index eceab1308a8c..8c250e8a2105 100644 --- a/Documentation/sparse.txt +++ b/Documentation/dev-tools/sparse.rst @@ -1,11 +1,20 @@ -Copyright 2004 Linus Torvalds -Copyright 2004 Pavel Machek -Copyright 2006 Bob Copeland +.. Copyright 2004 Linus Torvalds +.. Copyright 2004 Pavel Machek +.. Copyright 2006 Bob Copeland + +Sparse +====== + +Sparse is a semantic checker for C programs; it can be used to find a +number of potential problems with kernel code. See +https://lwn.net/Articles/689907/ for an overview of sparse; this document +contains some kernel-specific sparse information. + Using sparse for typechecking -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- -"__bitwise" is a type attribute, so you have to do something like this: +"__bitwise" is a type attribute, so you have to do something like this:: typedef int __bitwise pm_request_t; @@ -20,13 +29,13 @@ but in this case we really _do_ want to force the conversion). And because the enum values are all the same type, now "enum pm_request" will be that type too. -And with gcc, all the __bitwise/__force stuff goes away, and it all ends -up looking just like integers to gcc. +And with gcc, all the "__bitwise"/"__force stuff" goes away, and it all +ends up looking just like integers to gcc. Quite frankly, you don't need the enum there. The above all really just boils down to one special "int __bitwise" type. -So the simpler way is to just do +So the simpler way is to just do:: typedef int __bitwise pm_request_t; @@ -50,7 +59,7 @@ __bitwise - noisy stuff; in particular, __le*/__be* are that. We really don't want to drown in noise unless we'd explicitly asked for it. Using sparse for lock checking -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ The following macros are undefined for gcc and defined during a sparse run to use the "context" tracking feature of sparse, applied to @@ -69,22 +78,22 @@ annotation is needed. The tree annotations above are for cases where sparse would otherwise report a context imbalance. Getting sparse -~~~~~~~~~~~~~~ +-------------- You can get latest released versions from the Sparse homepage at https://sparse.wiki.kernel.org/index.php/Main_Page Alternatively, you can get snapshots of the latest development version -of sparse using git to clone.. +of sparse using git to clone:: git://git.kernel.org/pub/scm/devel/sparse/sparse.git -DaveJ has hourly generated tarballs of the git tree available at.. +DaveJ has hourly generated tarballs of the git tree available at:: http://www.codemonkey.org.uk/projects/git-snapshots/sparse/ -Once you have it, just do +Once you have it, just do:: make make install @@ -92,7 +101,7 @@ Once you have it, just do as a regular user, and it will install sparse in your ~/bin directory. Using sparse -~~~~~~~~~~~~ +------------ Do a kernel make with "make C=1" to run sparse on all the C files that get recompiled, or use "make C=2" to run sparse on the files whether they need to @@ -101,7 +110,7 @@ have already built it. The optional make variable CF can be used to pass arguments to sparse. The build system passes -Wbitwise to sparse automatically. To perform endianness -checks, you may define __CHECK_ENDIAN__: +checks, you may define __CHECK_ENDIAN__:: make C=2 CF="-D__CHECK_ENDIAN__" diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index ae0c58c784db..d4bbda319e79 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -15,3 +15,4 @@ whole; patches welcome! :maxdepth: 2 coccinelle + sparse From 758f726e7f7c5a09d8627e6e6ad914e568b31a5c Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:13:00 -0600 Subject: [PATCH 35/51] docs: sphinxify kcov.txt and move to dev-tools Another document added to the dev-tools collection. Cc: Dmitry Vyukov Signed-off-by: Jonathan Corbet --- .../{kcov.txt => dev-tools/kcov.rst} | 74 +++++++++---------- Documentation/dev-tools/tools.rst | 1 + 2 files changed, 38 insertions(+), 37 deletions(-) rename Documentation/{kcov.txt => dev-tools/kcov.rst} (78%) diff --git a/Documentation/kcov.txt b/Documentation/dev-tools/kcov.rst similarity index 78% rename from Documentation/kcov.txt rename to Documentation/dev-tools/kcov.rst index 779ff4ab1c1d..aca0e27ca197 100644 --- a/Documentation/kcov.txt +++ b/Documentation/dev-tools/kcov.rst @@ -12,38 +12,38 @@ To achieve this goal it does not collect coverage in soft/hard interrupts and instrumentation of some inherently non-deterministic parts of kernel is disbled (e.g. scheduler, locking). -Usage: -====== +Usage +----- -Configure kernel with: +Configure the kernel with:: CONFIG_KCOV=y CONFIG_KCOV requires gcc built on revision 231296 or later. -Profiling data will only become accessible once debugfs has been mounted: +Profiling data will only become accessible once debugfs has been mounted:: mount -t debugfs none /sys/kernel/debug -The following program demonstrates kcov usage from within a test program: +The following program demonstrates kcov usage from within a test program:: -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include -#define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) -#define KCOV_ENABLE _IO('c', 100) -#define KCOV_DISABLE _IO('c', 101) -#define COVER_SIZE (64<<10) + #define KCOV_INIT_TRACE _IOR('c', 1, unsigned long) + #define KCOV_ENABLE _IO('c', 100) + #define KCOV_DISABLE _IO('c', 101) + #define COVER_SIZE (64<<10) -int main(int argc, char **argv) -{ + int main(int argc, char **argv) + { int fd; unsigned long *cover, n, i; @@ -83,24 +83,24 @@ int main(int argc, char **argv) if (close(fd)) perror("close"), exit(1); return 0; -} + } -After piping through addr2line output of the program looks as follows: +After piping through addr2line output of the program looks as follows:: -SyS_read -fs/read_write.c:562 -__fdget_pos -fs/file.c:774 -__fget_light -fs/file.c:746 -__fget_light -fs/file.c:750 -__fget_light -fs/file.c:760 -__fdget_pos -fs/file.c:784 -SyS_read -fs/read_write.c:562 + SyS_read + fs/read_write.c:562 + __fdget_pos + fs/file.c:774 + __fget_light + fs/file.c:746 + __fget_light + fs/file.c:750 + __fget_light + fs/file.c:760 + __fdget_pos + fs/file.c:784 + SyS_read + fs/read_write.c:562 If a program needs to collect coverage from several threads (independently), it needs to open /sys/kernel/debug/kcov in each thread separately. diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index d4bbda319e79..9dcd0236a1ff 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -16,3 +16,4 @@ whole; patches welcome! coccinelle sparse + kcov From 2584bab2f990281244d3f39dc15cd9bd6d69f51d Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:26:20 -0600 Subject: [PATCH 36/51] docs: sphinixfy gcov.txt and move to dev-tools No textual changes beyond formatting. Cc: Peter Oberparleiter Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/gcov.rst | 256 +++++++++++++++++++++++++++++ Documentation/dev-tools/tools.rst | 1 + Documentation/gcov.txt | 257 ------------------------------ MAINTAINERS | 2 +- 4 files changed, 258 insertions(+), 258 deletions(-) create mode 100644 Documentation/dev-tools/gcov.rst delete mode 100644 Documentation/gcov.txt diff --git a/Documentation/dev-tools/gcov.rst b/Documentation/dev-tools/gcov.rst new file mode 100644 index 000000000000..19eedfea8800 --- /dev/null +++ b/Documentation/dev-tools/gcov.rst @@ -0,0 +1,256 @@ +Using gcov with the Linux kernel +================================ + +gcov profiling kernel support enables the use of GCC's coverage testing +tool gcov_ with the Linux kernel. Coverage data of a running kernel +is exported in gcov-compatible format via the "gcov" debugfs directory. +To get coverage data for a specific file, change to the kernel build +directory and use gcov with the ``-o`` option as follows (requires root):: + + # cd /tmp/linux-out + # gcov -o /sys/kernel/debug/gcov/tmp/linux-out/kernel spinlock.c + +This will create source code files annotated with execution counts +in the current directory. In addition, graphical gcov front-ends such +as lcov_ can be used to automate the process of collecting data +for the entire kernel and provide coverage overviews in HTML format. + +Possible uses: + +* debugging (has this line been reached at all?) +* test improvement (how do I change my test to cover these lines?) +* minimizing kernel configurations (do I need this option if the + associated code is never run?) + +.. _gcov: http://gcc.gnu.org/onlinedocs/gcc/Gcov.html +.. _lcov: http://ltp.sourceforge.net/coverage/lcov.php + + +Preparation +----------- + +Configure the kernel with:: + + CONFIG_DEBUG_FS=y + CONFIG_GCOV_KERNEL=y + +select the gcc's gcov format, default is autodetect based on gcc version:: + + CONFIG_GCOV_FORMAT_AUTODETECT=y + +and to get coverage data for the entire kernel:: + + CONFIG_GCOV_PROFILE_ALL=y + +Note that kernels compiled with profiling flags will be significantly +larger and run slower. Also CONFIG_GCOV_PROFILE_ALL may not be supported +on all architectures. + +Profiling data will only become accessible once debugfs has been +mounted:: + + mount -t debugfs none /sys/kernel/debug + + +Customization +------------- + +To enable profiling for specific files or directories, add a line +similar to the following to the respective kernel Makefile: + +- For a single file (e.g. main.o):: + + GCOV_PROFILE_main.o := y + +- For all files in one directory:: + + GCOV_PROFILE := y + +To exclude files from being profiled even when CONFIG_GCOV_PROFILE_ALL +is specified, use:: + + GCOV_PROFILE_main.o := n + +and:: + + GCOV_PROFILE := n + +Only files which are linked to the main kernel image or are compiled as +kernel modules are supported by this mechanism. + + +Files +----- + +The gcov kernel support creates the following files in debugfs: + +``/sys/kernel/debug/gcov`` + Parent directory for all gcov-related files. + +``/sys/kernel/debug/gcov/reset`` + Global reset file: resets all coverage data to zero when + written to. + +``/sys/kernel/debug/gcov/path/to/compile/dir/file.gcda`` + The actual gcov data file as understood by the gcov + tool. Resets file coverage data to zero when written to. + +``/sys/kernel/debug/gcov/path/to/compile/dir/file.gcno`` + Symbolic link to a static data file required by the gcov + tool. This file is generated by gcc when compiling with + option ``-ftest-coverage``. + + +Modules +------- + +Kernel modules may contain cleanup code which is only run during +module unload time. The gcov mechanism provides a means to collect +coverage data for such code by keeping a copy of the data associated +with the unloaded module. This data remains available through debugfs. +Once the module is loaded again, the associated coverage counters are +initialized with the data from its previous instantiation. + +This behavior can be deactivated by specifying the gcov_persist kernel +parameter:: + + gcov_persist=0 + +At run-time, a user can also choose to discard data for an unloaded +module by writing to its data file or the global reset file. + + +Separated build and test machines +--------------------------------- + +The gcov kernel profiling infrastructure is designed to work out-of-the +box for setups where kernels are built and run on the same machine. In +cases where the kernel runs on a separate machine, special preparations +must be made, depending on where the gcov tool is used: + +a) gcov is run on the TEST machine + + The gcov tool version on the test machine must be compatible with the + gcc version used for kernel build. Also the following files need to be + copied from build to test machine: + + from the source tree: + - all C source files + headers + + from the build tree: + - all C source files + headers + - all .gcda and .gcno files + - all links to directories + + It is important to note that these files need to be placed into the + exact same file system location on the test machine as on the build + machine. If any of the path components is symbolic link, the actual + directory needs to be used instead (due to make's CURDIR handling). + +b) gcov is run on the BUILD machine + + The following files need to be copied after each test case from test + to build machine: + + from the gcov directory in sysfs: + - all .gcda files + - all links to .gcno files + + These files can be copied to any location on the build machine. gcov + must then be called with the -o option pointing to that directory. + + Example directory setup on the build machine:: + + /tmp/linux: kernel source tree + /tmp/out: kernel build directory as specified by make O= + /tmp/coverage: location of the files copied from the test machine + + [user@build] cd /tmp/out + [user@build] gcov -o /tmp/coverage/tmp/out/init main.c + + +Troubleshooting +--------------- + +Problem + Compilation aborts during linker step. + +Cause + Profiling flags are specified for source files which are not + linked to the main kernel or which are linked by a custom + linker procedure. + +Solution + Exclude affected source files from profiling by specifying + ``GCOV_PROFILE := n`` or ``GCOV_PROFILE_basename.o := n`` in the + corresponding Makefile. + +Problem + Files copied from sysfs appear empty or incomplete. + +Cause + Due to the way seq_file works, some tools such as cp or tar + may not correctly copy files from sysfs. + +Solution + Use ``cat``' to read ``.gcda`` files and ``cp -d`` to copy links. + Alternatively use the mechanism shown in Appendix B. + + +Appendix A: gather_on_build.sh +------------------------------ + +Sample script to gather coverage meta files on the build machine +(see 6a):: + + #!/bin/bash + + KSRC=$1 + KOBJ=$2 + DEST=$3 + + if [ -z "$KSRC" ] || [ -z "$KOBJ" ] || [ -z "$DEST" ]; then + echo "Usage: $0 " >&2 + exit 1 + fi + + KSRC=$(cd $KSRC; printf "all:\n\t@echo \${CURDIR}\n" | make -f -) + KOBJ=$(cd $KOBJ; printf "all:\n\t@echo \${CURDIR}\n" | make -f -) + + find $KSRC $KOBJ \( -name '*.gcno' -o -name '*.[ch]' -o -type l \) -a \ + -perm /u+r,g+r | tar cfz $DEST -P -T - + + if [ $? -eq 0 ] ; then + echo "$DEST successfully created, copy to test system and unpack with:" + echo " tar xfz $DEST -P" + else + echo "Could not create file $DEST" + fi + + +Appendix B: gather_on_test.sh +----------------------------- + +Sample script to gather coverage data files on the test machine +(see 6b):: + + #!/bin/bash -e + + DEST=$1 + GCDA=/sys/kernel/debug/gcov + + if [ -z "$DEST" ] ; then + echo "Usage: $0 " >&2 + exit 1 + fi + + TEMPDIR=$(mktemp -d) + echo Collecting data.. + find $GCDA -type d -exec mkdir -p $TEMPDIR/\{\} \; + find $GCDA -name '*.gcda' -exec sh -c 'cat < $0 > '$TEMPDIR'/$0' {} \; + find $GCDA -name '*.gcno' -exec sh -c 'cp -d $0 '$TEMPDIR'/$0' {} \; + tar czf $DEST -C $TEMPDIR sys + rm -rf $TEMPDIR + + echo "$DEST successfully created, copy to build system and unpack with:" + echo " tar xfz $DEST" diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 9dcd0236a1ff..404d044c8518 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -17,3 +17,4 @@ whole; patches welcome! coccinelle sparse kcov + gcov diff --git a/Documentation/gcov.txt b/Documentation/gcov.txt deleted file mode 100644 index 7b727783db7e..000000000000 --- a/Documentation/gcov.txt +++ /dev/null @@ -1,257 +0,0 @@ -Using gcov with the Linux kernel -================================ - -1. Introduction -2. Preparation -3. Customization -4. Files -5. Modules -6. Separated build and test machines -7. Troubleshooting -Appendix A: sample script: gather_on_build.sh -Appendix B: sample script: gather_on_test.sh - - -1. Introduction -=============== - -gcov profiling kernel support enables the use of GCC's coverage testing -tool gcov [1] with the Linux kernel. Coverage data of a running kernel -is exported in gcov-compatible format via the "gcov" debugfs directory. -To get coverage data for a specific file, change to the kernel build -directory and use gcov with the -o option as follows (requires root): - -# cd /tmp/linux-out -# gcov -o /sys/kernel/debug/gcov/tmp/linux-out/kernel spinlock.c - -This will create source code files annotated with execution counts -in the current directory. In addition, graphical gcov front-ends such -as lcov [2] can be used to automate the process of collecting data -for the entire kernel and provide coverage overviews in HTML format. - -Possible uses: - -* debugging (has this line been reached at all?) -* test improvement (how do I change my test to cover these lines?) -* minimizing kernel configurations (do I need this option if the - associated code is never run?) - --- - -[1] http://gcc.gnu.org/onlinedocs/gcc/Gcov.html -[2] http://ltp.sourceforge.net/coverage/lcov.php - - -2. Preparation -============== - -Configure the kernel with: - - CONFIG_DEBUG_FS=y - CONFIG_GCOV_KERNEL=y - -select the gcc's gcov format, default is autodetect based on gcc version: - - CONFIG_GCOV_FORMAT_AUTODETECT=y - -and to get coverage data for the entire kernel: - - CONFIG_GCOV_PROFILE_ALL=y - -Note that kernels compiled with profiling flags will be significantly -larger and run slower. Also CONFIG_GCOV_PROFILE_ALL may not be supported -on all architectures. - -Profiling data will only become accessible once debugfs has been -mounted: - - mount -t debugfs none /sys/kernel/debug - - -3. Customization -================ - -To enable profiling for specific files or directories, add a line -similar to the following to the respective kernel Makefile: - - For a single file (e.g. main.o): - GCOV_PROFILE_main.o := y - - For all files in one directory: - GCOV_PROFILE := y - -To exclude files from being profiled even when CONFIG_GCOV_PROFILE_ALL -is specified, use: - - GCOV_PROFILE_main.o := n - and: - GCOV_PROFILE := n - -Only files which are linked to the main kernel image or are compiled as -kernel modules are supported by this mechanism. - - -4. Files -======== - -The gcov kernel support creates the following files in debugfs: - - /sys/kernel/debug/gcov - Parent directory for all gcov-related files. - - /sys/kernel/debug/gcov/reset - Global reset file: resets all coverage data to zero when - written to. - - /sys/kernel/debug/gcov/path/to/compile/dir/file.gcda - The actual gcov data file as understood by the gcov - tool. Resets file coverage data to zero when written to. - - /sys/kernel/debug/gcov/path/to/compile/dir/file.gcno - Symbolic link to a static data file required by the gcov - tool. This file is generated by gcc when compiling with - option -ftest-coverage. - - -5. Modules -========== - -Kernel modules may contain cleanup code which is only run during -module unload time. The gcov mechanism provides a means to collect -coverage data for such code by keeping a copy of the data associated -with the unloaded module. This data remains available through debugfs. -Once the module is loaded again, the associated coverage counters are -initialized with the data from its previous instantiation. - -This behavior can be deactivated by specifying the gcov_persist kernel -parameter: - - gcov_persist=0 - -At run-time, a user can also choose to discard data for an unloaded -module by writing to its data file or the global reset file. - - -6. Separated build and test machines -==================================== - -The gcov kernel profiling infrastructure is designed to work out-of-the -box for setups where kernels are built and run on the same machine. In -cases where the kernel runs on a separate machine, special preparations -must be made, depending on where the gcov tool is used: - -a) gcov is run on the TEST machine - -The gcov tool version on the test machine must be compatible with the -gcc version used for kernel build. Also the following files need to be -copied from build to test machine: - -from the source tree: - - all C source files + headers - -from the build tree: - - all C source files + headers - - all .gcda and .gcno files - - all links to directories - -It is important to note that these files need to be placed into the -exact same file system location on the test machine as on the build -machine. If any of the path components is symbolic link, the actual -directory needs to be used instead (due to make's CURDIR handling). - -b) gcov is run on the BUILD machine - -The following files need to be copied after each test case from test -to build machine: - -from the gcov directory in sysfs: - - all .gcda files - - all links to .gcno files - -These files can be copied to any location on the build machine. gcov -must then be called with the -o option pointing to that directory. - -Example directory setup on the build machine: - - /tmp/linux: kernel source tree - /tmp/out: kernel build directory as specified by make O= - /tmp/coverage: location of the files copied from the test machine - - [user@build] cd /tmp/out - [user@build] gcov -o /tmp/coverage/tmp/out/init main.c - - -7. Troubleshooting -================== - -Problem: Compilation aborts during linker step. -Cause: Profiling flags are specified for source files which are not - linked to the main kernel or which are linked by a custom - linker procedure. -Solution: Exclude affected source files from profiling by specifying - GCOV_PROFILE := n or GCOV_PROFILE_basename.o := n in the - corresponding Makefile. - -Problem: Files copied from sysfs appear empty or incomplete. -Cause: Due to the way seq_file works, some tools such as cp or tar - may not correctly copy files from sysfs. -Solution: Use 'cat' to read .gcda files and 'cp -d' to copy links. - Alternatively use the mechanism shown in Appendix B. - - -Appendix A: gather_on_build.sh -============================== - -Sample script to gather coverage meta files on the build machine -(see 6a): -#!/bin/bash - -KSRC=$1 -KOBJ=$2 -DEST=$3 - -if [ -z "$KSRC" ] || [ -z "$KOBJ" ] || [ -z "$DEST" ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -KSRC=$(cd $KSRC; printf "all:\n\t@echo \${CURDIR}\n" | make -f -) -KOBJ=$(cd $KOBJ; printf "all:\n\t@echo \${CURDIR}\n" | make -f -) - -find $KSRC $KOBJ \( -name '*.gcno' -o -name '*.[ch]' -o -type l \) -a \ - -perm /u+r,g+r | tar cfz $DEST -P -T - - -if [ $? -eq 0 ] ; then - echo "$DEST successfully created, copy to test system and unpack with:" - echo " tar xfz $DEST -P" -else - echo "Could not create file $DEST" -fi - - -Appendix B: gather_on_test.sh -============================= - -Sample script to gather coverage data files on the test machine -(see 6b): - -#!/bin/bash -e - -DEST=$1 -GCDA=/sys/kernel/debug/gcov - -if [ -z "$DEST" ] ; then - echo "Usage: $0 " >&2 - exit 1 -fi - -TEMPDIR=$(mktemp -d) -echo Collecting data.. -find $GCDA -type d -exec mkdir -p $TEMPDIR/\{\} \; -find $GCDA -name '*.gcda' -exec sh -c 'cat < $0 > '$TEMPDIR'/$0' {} \; -find $GCDA -name '*.gcno' -exec sh -c 'cp -d $0 '$TEMPDIR'/$0' {} \; -tar czf $DEST -C $TEMPDIR sys -rm -rf $TEMPDIR - -echo "$DEST successfully created, copy to build system and unpack with:" -echo " tar xfz $DEST" diff --git a/MAINTAINERS b/MAINTAINERS index 1e5460cfc196..bb5377957d84 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5118,7 +5118,7 @@ GCOV BASED KERNEL PROFILING M: Peter Oberparleiter S: Maintained F: kernel/gcov/ -F: Documentation/gcov.txt +F: Documentation/dev-tools/gcov.rst GDT SCSI DISK ARRAY CONTROLLER DRIVER M: Achim Leubner From 2757aafad9ee7b8740042e5540b59422337344ab Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:31:03 -0600 Subject: [PATCH 37/51] docs: sphinxify kasan.txt and move to dev-tools No textual changes beyond formatting. Acked-by: Andrey Ryabinin Acked-by: Alexander Potapenko Cc: Dmitry Vyukov Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/kasan.rst | 173 ++++++++++++++++++++++++++++++ Documentation/dev-tools/tools.rst | 1 + Documentation/kasan.txt | 171 ----------------------------- MAINTAINERS | 2 +- 4 files changed, 175 insertions(+), 172 deletions(-) create mode 100644 Documentation/dev-tools/kasan.rst delete mode 100644 Documentation/kasan.txt diff --git a/Documentation/dev-tools/kasan.rst b/Documentation/dev-tools/kasan.rst new file mode 100644 index 000000000000..948d243bed86 --- /dev/null +++ b/Documentation/dev-tools/kasan.rst @@ -0,0 +1,173 @@ +The Kernel Address Sanitizer (KASAN) +==================================== + +Overview +-------- + +KernelAddressSANitizer (KASAN) is a dynamic memory error detector. It provides +a fast and comprehensive solution for finding use-after-free and out-of-bounds +bugs. + +KASAN uses compile-time instrumentation for checking every memory access, +therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is +required for detection of out-of-bounds accesses to stack or global variables. + +Currently KASAN is supported only for x86_64 architecture. + +Usage +----- + +To enable KASAN configure kernel with:: + + CONFIG_KASAN = y + +and choose between CONFIG_KASAN_OUTLINE and CONFIG_KASAN_INLINE. Outline and +inline are compiler instrumentation types. The former produces smaller binary +the latter is 1.1 - 2 times faster. Inline instrumentation requires a GCC +version 5.0 or later. + +KASAN works with both SLUB and SLAB memory allocators. +For better bug detection and nicer reporting, enable CONFIG_STACKTRACE. + +To disable instrumentation for specific files or directories, add a line +similar to the following to the respective kernel Makefile: + +- For a single file (e.g. main.o):: + + KASAN_SANITIZE_main.o := n + +- For all files in one directory:: + + KASAN_SANITIZE := n + +Error reports +~~~~~~~~~~~~~ + +A typical out of bounds access report looks like this:: + + ================================================================== + BUG: AddressSanitizer: out of bounds access in kmalloc_oob_right+0x65/0x75 [test_kasan] at addr ffff8800693bc5d3 + Write of size 1 by task modprobe/1689 + ============================================================================= + BUG kmalloc-128 (Not tainted): kasan error + ----------------------------------------------------------------------------- + + Disabling lock debugging due to kernel taint + INFO: Allocated in kmalloc_oob_right+0x3d/0x75 [test_kasan] age=0 cpu=0 pid=1689 + __slab_alloc+0x4b4/0x4f0 + kmem_cache_alloc_trace+0x10b/0x190 + kmalloc_oob_right+0x3d/0x75 [test_kasan] + init_module+0x9/0x47 [test_kasan] + do_one_initcall+0x99/0x200 + load_module+0x2cb3/0x3b20 + SyS_finit_module+0x76/0x80 + system_call_fastpath+0x12/0x17 + INFO: Slab 0xffffea0001a4ef00 objects=17 used=7 fp=0xffff8800693bd728 flags=0x100000000004080 + INFO: Object 0xffff8800693bc558 @offset=1368 fp=0xffff8800693bc720 + + Bytes b4 ffff8800693bc548: 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ + Object ffff8800693bc558: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc568: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc578: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc588: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc598: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc5a8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc5b8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk + Object ffff8800693bc5c8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b a5 kkkkkkkkkkkkkkk. + Redzone ffff8800693bc5d8: cc cc cc cc cc cc cc cc ........ + Padding ffff8800693bc718: 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ + CPU: 0 PID: 1689 Comm: modprobe Tainted: G B 3.18.0-rc1-mm1+ #98 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.7.5-0-ge51488c-20140602_164612-nilsson.home.kraxel.org 04/01/2014 + ffff8800693bc000 0000000000000000 ffff8800693bc558 ffff88006923bb78 + ffffffff81cc68ae 00000000000000f3 ffff88006d407600 ffff88006923bba8 + ffffffff811fd848 ffff88006d407600 ffffea0001a4ef00 ffff8800693bc558 + Call Trace: + [] dump_stack+0x46/0x58 + [] print_trailer+0xf8/0x160 + [] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] + [] object_err+0x35/0x40 + [] ? kmalloc_oob_right+0x65/0x75 [test_kasan] + [] kasan_report_error+0x38a/0x3f0 + [] ? kasan_poison_shadow+0x2f/0x40 + [] ? kasan_unpoison_shadow+0x14/0x40 + [] ? kasan_poison_shadow+0x2f/0x40 + [] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] + [] __asan_store1+0x75/0xb0 + [] ? kmem_cache_oob+0x1d/0xc3 [test_kasan] + [] ? kmalloc_oob_right+0x65/0x75 [test_kasan] + [] kmalloc_oob_right+0x65/0x75 [test_kasan] + [] init_module+0x9/0x47 [test_kasan] + [] do_one_initcall+0x99/0x200 + [] ? __vunmap+0xec/0x160 + [] load_module+0x2cb3/0x3b20 + [] ? m_show+0x240/0x240 + [] SyS_finit_module+0x76/0x80 + [] system_call_fastpath+0x12/0x17 + Memory state around the buggy address: + ffff8800693bc300: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc380: fc fc 00 00 00 00 00 00 00 00 00 00 00 00 00 fc + ffff8800693bc400: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc480: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc500: fc fc fc fc fc fc fc fc fc fc fc 00 00 00 00 00 + >ffff8800693bc580: 00 00 00 00 00 00 00 00 00 00 03 fc fc fc fc fc + ^ + ffff8800693bc600: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc680: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc + ffff8800693bc700: fc fc fc fc fb fb fb fb fb fb fb fb fb fb fb fb + ffff8800693bc780: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb + ffff8800693bc800: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb + ================================================================== + +The header of the report discribe what kind of bug happened and what kind of +access caused it. It's followed by the description of the accessed slub object +(see 'SLUB Debug output' section in Documentation/vm/slub.txt for details) and +the description of the accessed memory page. + +In the last section the report shows memory state around the accessed address. +Reading this part requires some understanding of how KASAN works. + +The state of each 8 aligned bytes of memory is encoded in one shadow byte. +Those 8 bytes can be accessible, partially accessible, freed or be a redzone. +We use the following encoding for each shadow byte: 0 means that all 8 bytes +of the corresponding memory region are accessible; number N (1 <= N <= 7) means +that the first N bytes are accessible, and other (8 - N) bytes are not; +any negative value indicates that the entire 8-byte word is inaccessible. +We use different negative values to distinguish between different kinds of +inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). + +In the report above the arrows point to the shadow byte 03, which means that +the accessed address is partially accessible. + + +Implementation details +---------------------- + +From a high level, our approach to memory error detection is similar to that +of kmemcheck: use shadow memory to record whether each byte of memory is safe +to access, and use compile-time instrumentation to check shadow memory on each +memory access. + +AddressSanitizer dedicates 1/8 of kernel memory to its shadow memory +(e.g. 16TB to cover 128TB on x86_64) and uses direct mapping with a scale and +offset to translate a memory address to its corresponding shadow address. + +Here is the function which translates an address to its corresponding shadow +address:: + + static inline void *kasan_mem_to_shadow(const void *addr) + { + return ((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_SHADOW_OFFSET; + } + +where ``KASAN_SHADOW_SCALE_SHIFT = 3``. + +Compile-time instrumentation used for checking memory accesses. Compiler inserts +function calls (__asan_load*(addr), __asan_store*(addr)) before each memory +access of size 1, 2, 4, 8 or 16. These functions check whether memory access is +valid or not by checking corresponding shadow memory. + +GCC 5.0 has possibility to perform inline instrumentation. Instead of making +function calls GCC directly inserts the code to check the shadow memory. +This option significantly enlarges kernel but it gives x1.1-x2 performance +boost over outline instrumented kernel. diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 404d044c8518..0500e6500255 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -18,3 +18,4 @@ whole; patches welcome! sparse kcov gcov + kasan diff --git a/Documentation/kasan.txt b/Documentation/kasan.txt deleted file mode 100644 index 7dd95b35cd7c..000000000000 --- a/Documentation/kasan.txt +++ /dev/null @@ -1,171 +0,0 @@ -KernelAddressSanitizer (KASAN) -============================== - -0. Overview -=========== - -KernelAddressSANitizer (KASAN) is a dynamic memory error detector. It provides -a fast and comprehensive solution for finding use-after-free and out-of-bounds -bugs. - -KASAN uses compile-time instrumentation for checking every memory access, -therefore you will need a GCC version 4.9.2 or later. GCC 5.0 or later is -required for detection of out-of-bounds accesses to stack or global variables. - -Currently KASAN is supported only for x86_64 architecture. - -1. Usage -======== - -To enable KASAN configure kernel with: - - CONFIG_KASAN = y - -and choose between CONFIG_KASAN_OUTLINE and CONFIG_KASAN_INLINE. Outline and -inline are compiler instrumentation types. The former produces smaller binary -the latter is 1.1 - 2 times faster. Inline instrumentation requires a GCC -version 5.0 or later. - -KASAN works with both SLUB and SLAB memory allocators. -For better bug detection and nicer reporting, enable CONFIG_STACKTRACE. - -To disable instrumentation for specific files or directories, add a line -similar to the following to the respective kernel Makefile: - - For a single file (e.g. main.o): - KASAN_SANITIZE_main.o := n - - For all files in one directory: - KASAN_SANITIZE := n - -1.1 Error reports -================= - -A typical out of bounds access report looks like this: - -================================================================== -BUG: AddressSanitizer: out of bounds access in kmalloc_oob_right+0x65/0x75 [test_kasan] at addr ffff8800693bc5d3 -Write of size 1 by task modprobe/1689 -============================================================================= -BUG kmalloc-128 (Not tainted): kasan error ------------------------------------------------------------------------------ - -Disabling lock debugging due to kernel taint -INFO: Allocated in kmalloc_oob_right+0x3d/0x75 [test_kasan] age=0 cpu=0 pid=1689 - __slab_alloc+0x4b4/0x4f0 - kmem_cache_alloc_trace+0x10b/0x190 - kmalloc_oob_right+0x3d/0x75 [test_kasan] - init_module+0x9/0x47 [test_kasan] - do_one_initcall+0x99/0x200 - load_module+0x2cb3/0x3b20 - SyS_finit_module+0x76/0x80 - system_call_fastpath+0x12/0x17 -INFO: Slab 0xffffea0001a4ef00 objects=17 used=7 fp=0xffff8800693bd728 flags=0x100000000004080 -INFO: Object 0xffff8800693bc558 @offset=1368 fp=0xffff8800693bc720 - -Bytes b4 ffff8800693bc548: 00 00 00 00 00 00 00 00 5a 5a 5a 5a 5a 5a 5a 5a ........ZZZZZZZZ -Object ffff8800693bc558: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc568: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc578: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc588: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc598: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc5a8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc5b8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b kkkkkkkkkkkkkkkk -Object ffff8800693bc5c8: 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b 6b a5 kkkkkkkkkkkkkkk. -Redzone ffff8800693bc5d8: cc cc cc cc cc cc cc cc ........ -Padding ffff8800693bc718: 5a 5a 5a 5a 5a 5a 5a 5a ZZZZZZZZ -CPU: 0 PID: 1689 Comm: modprobe Tainted: G B 3.18.0-rc1-mm1+ #98 -Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS rel-1.7.5-0-ge51488c-20140602_164612-nilsson.home.kraxel.org 04/01/2014 - ffff8800693bc000 0000000000000000 ffff8800693bc558 ffff88006923bb78 - ffffffff81cc68ae 00000000000000f3 ffff88006d407600 ffff88006923bba8 - ffffffff811fd848 ffff88006d407600 ffffea0001a4ef00 ffff8800693bc558 -Call Trace: - [] dump_stack+0x46/0x58 - [] print_trailer+0xf8/0x160 - [] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] - [] object_err+0x35/0x40 - [] ? kmalloc_oob_right+0x65/0x75 [test_kasan] - [] kasan_report_error+0x38a/0x3f0 - [] ? kasan_poison_shadow+0x2f/0x40 - [] ? kasan_unpoison_shadow+0x14/0x40 - [] ? kasan_poison_shadow+0x2f/0x40 - [] ? kmem_cache_oob+0xc3/0xc3 [test_kasan] - [] __asan_store1+0x75/0xb0 - [] ? kmem_cache_oob+0x1d/0xc3 [test_kasan] - [] ? kmalloc_oob_right+0x65/0x75 [test_kasan] - [] kmalloc_oob_right+0x65/0x75 [test_kasan] - [] init_module+0x9/0x47 [test_kasan] - [] do_one_initcall+0x99/0x200 - [] ? __vunmap+0xec/0x160 - [] load_module+0x2cb3/0x3b20 - [] ? m_show+0x240/0x240 - [] SyS_finit_module+0x76/0x80 - [] system_call_fastpath+0x12/0x17 -Memory state around the buggy address: - ffff8800693bc300: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc - ffff8800693bc380: fc fc 00 00 00 00 00 00 00 00 00 00 00 00 00 fc - ffff8800693bc400: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc - ffff8800693bc480: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc - ffff8800693bc500: fc fc fc fc fc fc fc fc fc fc fc 00 00 00 00 00 ->ffff8800693bc580: 00 00 00 00 00 00 00 00 00 00 03 fc fc fc fc fc - ^ - ffff8800693bc600: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc - ffff8800693bc680: fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc fc - ffff8800693bc700: fc fc fc fc fb fb fb fb fb fb fb fb fb fb fb fb - ffff8800693bc780: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb - ffff8800693bc800: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb -================================================================== - -The header of the report discribe what kind of bug happened and what kind of -access caused it. It's followed by the description of the accessed slub object -(see 'SLUB Debug output' section in Documentation/vm/slub.txt for details) and -the description of the accessed memory page. - -In the last section the report shows memory state around the accessed address. -Reading this part requires some understanding of how KASAN works. - -The state of each 8 aligned bytes of memory is encoded in one shadow byte. -Those 8 bytes can be accessible, partially accessible, freed or be a redzone. -We use the following encoding for each shadow byte: 0 means that all 8 bytes -of the corresponding memory region are accessible; number N (1 <= N <= 7) means -that the first N bytes are accessible, and other (8 - N) bytes are not; -any negative value indicates that the entire 8-byte word is inaccessible. -We use different negative values to distinguish between different kinds of -inaccessible memory like redzones or freed memory (see mm/kasan/kasan.h). - -In the report above the arrows point to the shadow byte 03, which means that -the accessed address is partially accessible. - - -2. Implementation details -========================= - -From a high level, our approach to memory error detection is similar to that -of kmemcheck: use shadow memory to record whether each byte of memory is safe -to access, and use compile-time instrumentation to check shadow memory on each -memory access. - -AddressSanitizer dedicates 1/8 of kernel memory to its shadow memory -(e.g. 16TB to cover 128TB on x86_64) and uses direct mapping with a scale and -offset to translate a memory address to its corresponding shadow address. - -Here is the function which translates an address to its corresponding shadow -address: - -static inline void *kasan_mem_to_shadow(const void *addr) -{ - return ((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) - + KASAN_SHADOW_OFFSET; -} - -where KASAN_SHADOW_SCALE_SHIFT = 3. - -Compile-time instrumentation used for checking memory accesses. Compiler inserts -function calls (__asan_load*(addr), __asan_store*(addr)) before each memory -access of size 1, 2, 4, 8 or 16. These functions check whether memory access is -valid or not by checking corresponding shadow memory. - -GCC 5.0 has possibility to perform inline instrumentation. Instead of making -function calls GCC directly inserts the code to check the shadow memory. -This option significantly enlarges kernel but it gives x1.1-x2 performance -boost over outline instrumented kernel. diff --git a/MAINTAINERS b/MAINTAINERS index bb5377957d84..2ffd7ed40162 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6587,7 +6587,7 @@ L: kasan-dev@googlegroups.com S: Maintained F: arch/*/include/asm/kasan.h F: arch/*/mm/kasan_init* -F: Documentation/kasan.txt +F: Documentation/dev-tools/kasan.rst F: include/linux/kasan*.h F: lib/test_kasan.c F: mm/kasan/ From 1ead009cd622bc4c3c2cf1036d8e71d7f063838e Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:35:42 -0600 Subject: [PATCH 38/51] docs: sphinxify ubsan.txt and move it to dev-tools Acked-by: Andrey Ryabinin Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/tools.rst | 1 + .../{ubsan.txt => dev-tools/ubsan.rst} | 42 ++++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) rename Documentation/{ubsan.txt => dev-tools/ubsan.rst} (78%) diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 0500e6500255..2d1129789753 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -19,3 +19,4 @@ whole; patches welcome! kcov gcov kasan + ubsan diff --git a/Documentation/ubsan.txt b/Documentation/dev-tools/ubsan.rst similarity index 78% rename from Documentation/ubsan.txt rename to Documentation/dev-tools/ubsan.rst index f58215ef5797..655e6b63c227 100644 --- a/Documentation/ubsan.txt +++ b/Documentation/dev-tools/ubsan.rst @@ -1,7 +1,5 @@ -Undefined Behavior Sanitizer - UBSAN - -Overview --------- +The Undefined Behavior Sanitizer - UBSAN +======================================== UBSAN is a runtime undefined behaviour checker. @@ -10,11 +8,13 @@ Compiler inserts code that perform certain kinds of checks before operations that may cause UB. If check fails (i.e. UB detected) __ubsan_handle_* function called to print error message. -GCC has that feature since 4.9.x [1] (see -fsanitize=undefined option and -its suboptions). GCC 5.x has more checkers implemented [2]. +GCC has that feature since 4.9.x [1_] (see ``-fsanitize=undefined`` option and +its suboptions). GCC 5.x has more checkers implemented [2_]. Report example ---------------- +-------------- + +:: ================================================================================ UBSAN: Undefined behaviour in ../include/linux/bitops.h:110:33 @@ -47,29 +47,33 @@ Report example Usage ----- -To enable UBSAN configure kernel with: +To enable UBSAN configure kernel with:: CONFIG_UBSAN=y -and to check the entire kernel: +and to check the entire kernel:: CONFIG_UBSAN_SANITIZE_ALL=y To enable instrumentation for specific files or directories, add a line similar to the following to the respective kernel Makefile: - For a single file (e.g. main.o): - UBSAN_SANITIZE_main.o := y +- For a single file (e.g. main.o):: - For all files in one directory: - UBSAN_SANITIZE := y + UBSAN_SANITIZE_main.o := y + +- For all files in one directory:: + + UBSAN_SANITIZE := y To exclude files from being instrumented even if -CONFIG_UBSAN_SANITIZE_ALL=y, use: +``CONFIG_UBSAN_SANITIZE_ALL=y``, use:: - UBSAN_SANITIZE_main.o := n - and: - UBSAN_SANITIZE := n + UBSAN_SANITIZE_main.o := n + +and:: + + UBSAN_SANITIZE := n Detection of unaligned accesses controlled through the separate option - CONFIG_UBSAN_ALIGNMENT. It's off by default on architectures that support @@ -80,5 +84,5 @@ reports. References ---------- -[1] - https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/Debugging-Options.html -[2] - https://gcc.gnu.org/onlinedocs/gcc/Debugging-Options.html +.. _1: https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/Debugging-Options.html +.. _2: https://gcc.gnu.org/onlinedocs/gcc/Debugging-Options.html From ca90a7a38741adf5ce450572952fbbda35055ea4 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 15:46:10 -0600 Subject: [PATCH 39/51] docs: sphinxify kmemleak.txt and move it to dev-tools Acked-by: Catalin Marinas Signed-off-by: Jonathan Corbet --- .../{kmemleak.txt => dev-tools/kmemleak.rst} | 89 ++++++++++--------- Documentation/dev-tools/tools.rst | 1 + MAINTAINERS | 2 +- 3 files changed, 50 insertions(+), 42 deletions(-) rename Documentation/{kmemleak.txt => dev-tools/kmemleak.rst} (73%) diff --git a/Documentation/kmemleak.txt b/Documentation/dev-tools/kmemleak.rst similarity index 73% rename from Documentation/kmemleak.txt rename to Documentation/dev-tools/kmemleak.rst index 18e24abb3ecf..1788722d5495 100644 --- a/Documentation/kmemleak.txt +++ b/Documentation/dev-tools/kmemleak.rst @@ -1,15 +1,12 @@ Kernel Memory Leak Detector =========================== -Introduction ------------- - Kmemleak provides a way of detecting possible kernel memory leaks in a way similar to a tracing garbage collector (https://en.wikipedia.org/wiki/Garbage_collection_%28computer_science%29#Tracing_garbage_collectors), with the difference that the orphan objects are not freed but only reported via /sys/kernel/debug/kmemleak. A similar method is used by the -Valgrind tool (memcheck --leak-check) to detect the memory leaks in +Valgrind tool (``memcheck --leak-check``) to detect the memory leaks in user-space applications. Kmemleak is supported on x86, arm, powerpc, sparc, sh, microblaze, ppc, mips, s390, metag and tile. @@ -19,20 +16,20 @@ Usage CONFIG_DEBUG_KMEMLEAK in "Kernel hacking" has to be enabled. A kernel thread scans the memory every 10 minutes (by default) and prints the number of new unreferenced objects found. To display the details of all -the possible memory leaks: +the possible memory leaks:: # mount -t debugfs nodev /sys/kernel/debug/ # cat /sys/kernel/debug/kmemleak -To trigger an intermediate memory scan: +To trigger an intermediate memory scan:: # echo scan > /sys/kernel/debug/kmemleak -To clear the list of all current possible memory leaks: +To clear the list of all current possible memory leaks:: # echo clear > /sys/kernel/debug/kmemleak -New leaks will then come up upon reading /sys/kernel/debug/kmemleak +New leaks will then come up upon reading ``/sys/kernel/debug/kmemleak`` again. Note that the orphan objects are listed in the order they were allocated @@ -40,22 +37,31 @@ and one object at the beginning of the list may cause other subsequent objects to be reported as orphan. Memory scanning parameters can be modified at run-time by writing to the -/sys/kernel/debug/kmemleak file. The following parameters are supported: +``/sys/kernel/debug/kmemleak`` file. The following parameters are supported: - off - disable kmemleak (irreversible) - stack=on - enable the task stacks scanning (default) - stack=off - disable the tasks stacks scanning - scan=on - start the automatic memory scanning thread (default) - scan=off - stop the automatic memory scanning thread - scan= - set the automatic memory scanning period in seconds - (default 600, 0 to stop the automatic scanning) - scan - trigger a memory scan - clear - clear list of current memory leak suspects, done by - marking all current reported unreferenced objects grey, - or free all kmemleak objects if kmemleak has been disabled. - dump= - dump information about the object found at +- off + disable kmemleak (irreversible) +- stack=on + enable the task stacks scanning (default) +- stack=off + disable the tasks stacks scanning +- scan=on + start the automatic memory scanning thread (default) +- scan=off + stop the automatic memory scanning thread +- scan= + set the automatic memory scanning period in seconds + (default 600, 0 to stop the automatic scanning) +- scan + trigger a memory scan +- clear + clear list of current memory leak suspects, done by + marking all current reported unreferenced objects grey, + or free all kmemleak objects if kmemleak has been disabled. +- dump= + dump information about the object found at -Kmemleak can also be disabled at boot-time by passing "kmemleak=off" on +Kmemleak can also be disabled at boot-time by passing ``kmemleak=off`` on the kernel command line. Memory may be allocated or freed before kmemleak is initialised and @@ -63,13 +69,14 @@ these actions are stored in an early log buffer. The size of this buffer is configured via the CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE option. If CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF are enabled, the kmemleak is -disabled by default. Passing "kmemleak=on" on the kernel command +disabled by default. Passing ``kmemleak=on`` on the kernel command line enables the function. Basic Algorithm --------------- -The memory allocations via kmalloc, vmalloc, kmem_cache_alloc and +The memory allocations via :c:func:`kmalloc`, :c:func:`vmalloc`, +:c:func:`kmem_cache_alloc` and friends are traced and the pointers, together with additional information like size and stack trace, are stored in a rbtree. The corresponding freeing function calls are tracked and the pointers @@ -113,13 +120,13 @@ when doing development. To work around these situations you can use the you can find new unreferenced objects; this should help with testing specific sections of code. -To test a critical section on demand with a clean kmemleak do: +To test a critical section on demand with a clean kmemleak do:: # echo clear > /sys/kernel/debug/kmemleak ... test your kernel or modules ... # echo scan > /sys/kernel/debug/kmemleak -Then as usual to get your report with: +Then as usual to get your report with:: # cat /sys/kernel/debug/kmemleak @@ -131,7 +138,7 @@ disabled by the user or due to an fatal error, internal kmemleak objects won't be freed when kmemleak is disabled, and those objects may occupy a large part of physical memory. -In this situation, you may reclaim memory with: +In this situation, you may reclaim memory with:: # echo clear > /sys/kernel/debug/kmemleak @@ -140,20 +147,20 @@ Kmemleak API See the include/linux/kmemleak.h header for the functions prototype. -kmemleak_init - initialize kmemleak -kmemleak_alloc - notify of a memory block allocation -kmemleak_alloc_percpu - notify of a percpu memory block allocation -kmemleak_free - notify of a memory block freeing -kmemleak_free_part - notify of a partial memory block freeing -kmemleak_free_percpu - notify of a percpu memory block freeing -kmemleak_update_trace - update object allocation stack trace -kmemleak_not_leak - mark an object as not a leak -kmemleak_ignore - do not scan or report an object as leak -kmemleak_scan_area - add scan areas inside a memory block -kmemleak_no_scan - do not scan a memory block -kmemleak_erase - erase an old value in a pointer variable -kmemleak_alloc_recursive - as kmemleak_alloc but checks the recursiveness -kmemleak_free_recursive - as kmemleak_free but checks the recursiveness +- ``kmemleak_init`` - initialize kmemleak +- ``kmemleak_alloc`` - notify of a memory block allocation +- ``kmemleak_alloc_percpu`` - notify of a percpu memory block allocation +- ``kmemleak_free`` - notify of a memory block freeing +- ``kmemleak_free_part`` - notify of a partial memory block freeing +- ``kmemleak_free_percpu`` - notify of a percpu memory block freeing +- ``kmemleak_update_trace`` - update object allocation stack trace +- ``kmemleak_not_leak`` - mark an object as not a leak +- ``kmemleak_ignore`` - do not scan or report an object as leak +- ``kmemleak_scan_area`` - add scan areas inside a memory block +- ``kmemleak_no_scan`` - do not scan a memory block +- ``kmemleak_erase`` - erase an old value in a pointer variable +- ``kmemleak_alloc_recursive`` - as kmemleak_alloc but checks the recursiveness +- ``kmemleak_free_recursive`` - as kmemleak_free but checks the recursiveness Dealing with false positives/negatives -------------------------------------- diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 2d1129789753..3b6382a58301 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -20,3 +20,4 @@ whole; patches welcome! gcov kasan ubsan + kmemleak diff --git a/MAINTAINERS b/MAINTAINERS index 2ffd7ed40162..b235e0d53449 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6812,7 +6812,7 @@ F: mm/kmemcheck.c KMEMLEAK M: Catalin Marinas S: Maintained -F: Documentation/kmemleak.txt +F: Documentation/dev-tools/kmemleak.rst F: include/linux/kmemleak.h F: mm/kmemleak.c F: mm/kmemleak-test.c From 9c296b46c67ed323538bca80a8be1f0b10e570a0 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 7 Aug 2016 16:12:28 -0600 Subject: [PATCH 40/51] docs: sphinxify kmemcheck.txt and move to dev-tools Cc: Vegard Nossum Cc: Pekka Enberg Signed-off-by: Jonathan Corbet --- Documentation/dev-tools/kmemcheck.rst | 733 +++++++++++++++++++++++++ Documentation/dev-tools/tools.rst | 1 + Documentation/kmemcheck.txt | 754 -------------------------- MAINTAINERS | 2 +- 4 files changed, 735 insertions(+), 755 deletions(-) create mode 100644 Documentation/dev-tools/kmemcheck.rst delete mode 100644 Documentation/kmemcheck.txt diff --git a/Documentation/dev-tools/kmemcheck.rst b/Documentation/dev-tools/kmemcheck.rst new file mode 100644 index 000000000000..7f3d1985de74 --- /dev/null +++ b/Documentation/dev-tools/kmemcheck.rst @@ -0,0 +1,733 @@ +Getting started with kmemcheck +============================== + +Vegard Nossum + + +Introduction +------------ + +kmemcheck is a debugging feature for the Linux Kernel. More specifically, it +is a dynamic checker that detects and warns about some uses of uninitialized +memory. + +Userspace programmers might be familiar with Valgrind's memcheck. The main +difference between memcheck and kmemcheck is that memcheck works for userspace +programs only, and kmemcheck works for the kernel only. The implementations +are of course vastly different. Because of this, kmemcheck is not as accurate +as memcheck, but it turns out to be good enough in practice to discover real +programmer errors that the compiler is not able to find through static +analysis. + +Enabling kmemcheck on a kernel will probably slow it down to the extent that +the machine will not be usable for normal workloads such as e.g. an +interactive desktop. kmemcheck will also cause the kernel to use about twice +as much memory as normal. For this reason, kmemcheck is strictly a debugging +feature. + + +Downloading +----------- + +As of version 2.6.31-rc1, kmemcheck is included in the mainline kernel. + + +Configuring and compiling +------------------------- + +kmemcheck only works for the x86 (both 32- and 64-bit) platform. A number of +configuration variables must have specific settings in order for the kmemcheck +menu to even appear in "menuconfig". These are: + +- ``CONFIG_CC_OPTIMIZE_FOR_SIZE=n`` + This option is located under "General setup" / "Optimize for size". + + Without this, gcc will use certain optimizations that usually lead to + false positive warnings from kmemcheck. An example of this is a 16-bit + field in a struct, where gcc may load 32 bits, then discard the upper + 16 bits. kmemcheck sees only the 32-bit load, and may trigger a + warning for the upper 16 bits (if they're uninitialized). + +- ``CONFIG_SLAB=y`` or ``CONFIG_SLUB=y`` + This option is located under "General setup" / "Choose SLAB + allocator". + +- ``CONFIG_FUNCTION_TRACER=n`` + This option is located under "Kernel hacking" / "Tracers" / "Kernel + Function Tracer" + + When function tracing is compiled in, gcc emits a call to another + function at the beginning of every function. This means that when the + page fault handler is called, the ftrace framework will be called + before kmemcheck has had a chance to handle the fault. If ftrace then + modifies memory that was tracked by kmemcheck, the result is an + endless recursive page fault. + +- ``CONFIG_DEBUG_PAGEALLOC=n`` + This option is located under "Kernel hacking" / "Memory Debugging" + / "Debug page memory allocations". + +In addition, I highly recommend turning on ``CONFIG_DEBUG_INFO=y``. This is also +located under "Kernel hacking". With this, you will be able to get line number +information from the kmemcheck warnings, which is extremely valuable in +debugging a problem. This option is not mandatory, however, because it slows +down the compilation process and produces a much bigger kernel image. + +Now the kmemcheck menu should be visible (under "Kernel hacking" / "Memory +Debugging" / "kmemcheck: trap use of uninitialized memory"). Here follows +a description of the kmemcheck configuration variables: + +- ``CONFIG_KMEMCHECK`` + This must be enabled in order to use kmemcheck at all... + +- ``CONFIG_KMEMCHECK_``[``DISABLED`` | ``ENABLED`` | ``ONESHOT``]``_BY_DEFAULT`` + This option controls the status of kmemcheck at boot-time. "Enabled" + will enable kmemcheck right from the start, "disabled" will boot the + kernel as normal (but with the kmemcheck code compiled in, so it can + be enabled at run-time after the kernel has booted), and "one-shot" is + a special mode which will turn kmemcheck off automatically after + detecting the first use of uninitialized memory. + + If you are using kmemcheck to actively debug a problem, then you + probably want to choose "enabled" here. + + The one-shot mode is mostly useful in automated test setups because it + can prevent floods of warnings and increase the chances of the machine + surviving in case something is really wrong. In other cases, the one- + shot mode could actually be counter-productive because it would turn + itself off at the very first error -- in the case of a false positive + too -- and this would come in the way of debugging the specific + problem you were interested in. + + If you would like to use your kernel as normal, but with a chance to + enable kmemcheck in case of some problem, it might be a good idea to + choose "disabled" here. When kmemcheck is disabled, most of the run- + time overhead is not incurred, and the kernel will be almost as fast + as normal. + +- ``CONFIG_KMEMCHECK_QUEUE_SIZE`` + Select the maximum number of error reports to store in an internal + (fixed-size) buffer. Since errors can occur virtually anywhere and in + any context, we need a temporary storage area which is guaranteed not + to generate any other page faults when accessed. The queue will be + emptied as soon as a tasklet may be scheduled. If the queue is full, + new error reports will be lost. + + The default value of 64 is probably fine. If some code produces more + than 64 errors within an irqs-off section, then the code is likely to + produce many, many more, too, and these additional reports seldom give + any more information (the first report is usually the most valuable + anyway). + + This number might have to be adjusted if you are not using serial + console or similar to capture the kernel log. If you are using the + "dmesg" command to save the log, then getting a lot of kmemcheck + warnings might overflow the kernel log itself, and the earlier reports + will get lost in that way instead. Try setting this to 10 or so on + such a setup. + +- ``CONFIG_KMEMCHECK_SHADOW_COPY_SHIFT`` + Select the number of shadow bytes to save along with each entry of the + error-report queue. These bytes indicate what parts of an allocation + are initialized, uninitialized, etc. and will be displayed when an + error is detected to help the debugging of a particular problem. + + The number entered here is actually the logarithm of the number of + bytes that will be saved. So if you pick for example 5 here, kmemcheck + will save 2^5 = 32 bytes. + + The default value should be fine for debugging most problems. It also + fits nicely within 80 columns. + +- ``CONFIG_KMEMCHECK_PARTIAL_OK`` + This option (when enabled) works around certain GCC optimizations that + produce 32-bit reads from 16-bit variables where the upper 16 bits are + thrown away afterwards. + + The default value (enabled) is recommended. This may of course hide + some real errors, but disabling it would probably produce a lot of + false positives. + +- ``CONFIG_KMEMCHECK_BITOPS_OK`` + This option silences warnings that would be generated for bit-field + accesses where not all the bits are initialized at the same time. This + may also hide some real bugs. + + This option is probably obsolete, or it should be replaced with + the kmemcheck-/bitfield-annotations for the code in question. The + default value is therefore fine. + +Now compile the kernel as usual. + + +How to use +---------- + +Booting +~~~~~~~ + +First some information about the command-line options. There is only one +option specific to kmemcheck, and this is called "kmemcheck". It can be used +to override the default mode as chosen by the ``CONFIG_KMEMCHECK_*_BY_DEFAULT`` +option. Its possible settings are: + +- ``kmemcheck=0`` (disabled) +- ``kmemcheck=1`` (enabled) +- ``kmemcheck=2`` (one-shot mode) + +If SLUB debugging has been enabled in the kernel, it may take precedence over +kmemcheck in such a way that the slab caches which are under SLUB debugging +will not be tracked by kmemcheck. In order to ensure that this doesn't happen +(even though it shouldn't by default), use SLUB's boot option ``slub_debug``, +like this: ``slub_debug=-`` + +In fact, this option may also be used for fine-grained control over SLUB vs. +kmemcheck. For example, if the command line includes +``kmemcheck=1 slub_debug=,dentry``, then SLUB debugging will be used only +for the "dentry" slab cache, and with kmemcheck tracking all the other +caches. This is advanced usage, however, and is not generally recommended. + + +Run-time enable/disable +~~~~~~~~~~~~~~~~~~~~~~~ + +When the kernel has booted, it is possible to enable or disable kmemcheck at +run-time. WARNING: This feature is still experimental and may cause false +positive warnings to appear. Therefore, try not to use this. If you find that +it doesn't work properly (e.g. you see an unreasonable amount of warnings), I +will be happy to take bug reports. + +Use the file ``/proc/sys/kernel/kmemcheck`` for this purpose, e.g.:: + + $ echo 0 > /proc/sys/kernel/kmemcheck # disables kmemcheck + +The numbers are the same as for the ``kmemcheck=`` command-line option. + + +Debugging +~~~~~~~~~ + +A typical report will look something like this:: + + WARNING: kmemcheck: Caught 32-bit read from uninitialized memory (ffff88003e4a2024) + 80000000000000000000000000000000000000000088ffff0000000000000000 + i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u + ^ + + Pid: 1856, comm: ntpdate Not tainted 2.6.29-rc5 #264 945P-A + RIP: 0010:[] [] __dequeue_signal+0xc8/0x190 + RSP: 0018:ffff88003cdf7d98 EFLAGS: 00210002 + RAX: 0000000000000030 RBX: ffff88003d4ea968 RCX: 0000000000000009 + RDX: ffff88003e5d6018 RSI: ffff88003e5d6024 RDI: ffff88003cdf7e84 + RBP: ffff88003cdf7db8 R08: ffff88003e5d6000 R09: 0000000000000000 + R10: 0000000000000080 R11: 0000000000000000 R12: 000000000000000e + R13: ffff88003cdf7e78 R14: ffff88003d530710 R15: ffff88003d5a98c8 + FS: 0000000000000000(0000) GS:ffff880001982000(0063) knlGS:00000 + CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 + CR2: ffff88003f806ea0 CR3: 000000003c036000 CR4: 00000000000006a0 + DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 + DR3: 0000000000000000 DR6: 00000000ffff4ff0 DR7: 0000000000000400 + [] dequeue_signal+0x8e/0x170 + [] get_signal_to_deliver+0x98/0x390 + [] do_notify_resume+0xad/0x7d0 + [] int_signal+0x12/0x17 + [] 0xffffffffffffffff + +The single most valuable information in this report is the RIP (or EIP on 32- +bit) value. This will help us pinpoint exactly which instruction that caused +the warning. + +If your kernel was compiled with ``CONFIG_DEBUG_INFO=y``, then all we have to do +is give this address to the addr2line program, like this:: + + $ addr2line -e vmlinux -i ffffffff8104ede8 + arch/x86/include/asm/string_64.h:12 + include/asm-generic/siginfo.h:287 + kernel/signal.c:380 + kernel/signal.c:410 + +The "``-e vmlinux``" tells addr2line which file to look in. **IMPORTANT:** +This must be the vmlinux of the kernel that produced the warning in the +first place! If not, the line number information will almost certainly be +wrong. + +The "``-i``" tells addr2line to also print the line numbers of inlined +functions. In this case, the flag was very important, because otherwise, +it would only have printed the first line, which is just a call to +``memcpy()``, which could be called from a thousand places in the kernel, and +is therefore not very useful. These inlined functions would not show up in +the stack trace above, simply because the kernel doesn't load the extra +debugging information. This technique can of course be used with ordinary +kernel oopses as well. + +In this case, it's the caller of ``memcpy()`` that is interesting, and it can be +found in ``include/asm-generic/siginfo.h``, line 287:: + + 281 static inline void copy_siginfo(struct siginfo *to, struct siginfo *from) + 282 { + 283 if (from->si_code < 0) + 284 memcpy(to, from, sizeof(*to)); + 285 else + 286 /* _sigchld is currently the largest know union member */ + 287 memcpy(to, from, __ARCH_SI_PREAMBLE_SIZE + sizeof(from->_sifields._sigchld)); + 288 } + +Since this was a read (kmemcheck usually warns about reads only, though it can +warn about writes to unallocated or freed memory as well), it was probably the +"from" argument which contained some uninitialized bytes. Following the chain +of calls, we move upwards to see where "from" was allocated or initialized, +``kernel/signal.c``, line 380:: + + 359 static void collect_signal(int sig, struct sigpending *list, siginfo_t *info) + 360 { + ... + 367 list_for_each_entry(q, &list->list, list) { + 368 if (q->info.si_signo == sig) { + 369 if (first) + 370 goto still_pending; + 371 first = q; + ... + 377 if (first) { + 378 still_pending: + 379 list_del_init(&first->list); + 380 copy_siginfo(info, &first->info); + 381 __sigqueue_free(first); + ... + 392 } + 393 } + +Here, it is ``&first->info`` that is being passed on to ``copy_siginfo()``. The +variable ``first`` was found on a list -- passed in as the second argument to +``collect_signal()``. We continue our journey through the stack, to figure out +where the item on "list" was allocated or initialized. We move to line 410:: + + 395 static int __dequeue_signal(struct sigpending *pending, sigset_t *mask, + 396 siginfo_t *info) + 397 { + ... + 410 collect_signal(sig, pending, info); + ... + 414 } + +Now we need to follow the ``pending`` pointer, since that is being passed on to +``collect_signal()`` as ``list``. At this point, we've run out of lines from the +"addr2line" output. Not to worry, we just paste the next addresses from the +kmemcheck stack dump, i.e.:: + + [] dequeue_signal+0x8e/0x170 + [] get_signal_to_deliver+0x98/0x390 + [] do_notify_resume+0xad/0x7d0 + [] int_signal+0x12/0x17 + + $ addr2line -e vmlinux -i ffffffff8104f04e ffffffff81050bd8 \ + ffffffff8100b87d ffffffff8100c7b5 + kernel/signal.c:446 + kernel/signal.c:1806 + arch/x86/kernel/signal.c:805 + arch/x86/kernel/signal.c:871 + arch/x86/kernel/entry_64.S:694 + +Remember that since these addresses were found on the stack and not as the +RIP value, they actually point to the _next_ instruction (they are return +addresses). This becomes obvious when we look at the code for line 446:: + + 422 int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info) + 423 { + ... + 431 signr = __dequeue_signal(&tsk->signal->shared_pending, + 432 mask, info); + 433 /* + 434 * itimer signal ? + 435 * + 436 * itimers are process shared and we restart periodic + 437 * itimers in the signal delivery path to prevent DoS + 438 * attacks in the high resolution timer case. This is + 439 * compliant with the old way of self restarting + 440 * itimers, as the SIGALRM is a legacy signal and only + 441 * queued once. Changing the restart behaviour to + 442 * restart the timer in the signal dequeue path is + 443 * reducing the timer noise on heavy loaded !highres + 444 * systems too. + 445 */ + 446 if (unlikely(signr == SIGALRM)) { + ... + 489 } + +So instead of looking at 446, we should be looking at 431, which is the line +that executes just before 446. Here we see that what we are looking for is +``&tsk->signal->shared_pending``. + +Our next task is now to figure out which function that puts items on this +``shared_pending`` list. A crude, but efficient tool, is ``git grep``:: + + $ git grep -n 'shared_pending' kernel/ + ... + kernel/signal.c:828: pending = group ? &t->signal->shared_pending : &t->pending; + kernel/signal.c:1339: pending = group ? &t->signal->shared_pending : &t->pending; + ... + +There were more results, but none of them were related to list operations, +and these were the only assignments. We inspect the line numbers more closely +and find that this is indeed where items are being added to the list:: + + 816 static int send_signal(int sig, struct siginfo *info, struct task_struct *t, + 817 int group) + 818 { + ... + 828 pending = group ? &t->signal->shared_pending : &t->pending; + ... + 851 q = __sigqueue_alloc(t, GFP_ATOMIC, (sig < SIGRTMIN && + 852 (is_si_special(info) || + 853 info->si_code >= 0))); + 854 if (q) { + 855 list_add_tail(&q->list, &pending->list); + ... + 890 } + +and:: + + 1309 int send_sigqueue(struct sigqueue *q, struct task_struct *t, int group) + 1310 { + .... + 1339 pending = group ? &t->signal->shared_pending : &t->pending; + 1340 list_add_tail(&q->list, &pending->list); + .... + 1347 } + +In the first case, the list element we are looking for, ``q``, is being +returned from the function ``__sigqueue_alloc()``, which looks like an +allocation function. Let's take a look at it:: + + 187 static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags, + 188 int override_rlimit) + 189 { + 190 struct sigqueue *q = NULL; + 191 struct user_struct *user; + 192 + 193 /* + 194 * We won't get problems with the target's UID changing under us + 195 * because changing it requires RCU be used, and if t != current, the + 196 * caller must be holding the RCU readlock (by way of a spinlock) and + 197 * we use RCU protection here + 198 */ + 199 user = get_uid(__task_cred(t)->user); + 200 atomic_inc(&user->sigpending); + 201 if (override_rlimit || + 202 atomic_read(&user->sigpending) <= + 203 t->signal->rlim[RLIMIT_SIGPENDING].rlim_cur) + 204 q = kmem_cache_alloc(sigqueue_cachep, flags); + 205 if (unlikely(q == NULL)) { + 206 atomic_dec(&user->sigpending); + 207 free_uid(user); + 208 } else { + 209 INIT_LIST_HEAD(&q->list); + 210 q->flags = 0; + 211 q->user = user; + 212 } + 213 + 214 return q; + 215 } + +We see that this function initializes ``q->list``, ``q->flags``, and +``q->user``. It seems that now is the time to look at the definition of +``struct sigqueue``, e.g.:: + + 14 struct sigqueue { + 15 struct list_head list; + 16 int flags; + 17 siginfo_t info; + 18 struct user_struct *user; + 19 }; + +And, you might remember, it was a ``memcpy()`` on ``&first->info`` that +caused the warning, so this makes perfect sense. It also seems reasonable +to assume that it is the caller of ``__sigqueue_alloc()`` that has the +responsibility of filling out (initializing) this member. + +But just which fields of the struct were uninitialized? Let's look at +kmemcheck's report again:: + + WARNING: kmemcheck: Caught 32-bit read from uninitialized memory (ffff88003e4a2024) + 80000000000000000000000000000000000000000088ffff0000000000000000 + i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u + ^ + +These first two lines are the memory dump of the memory object itself, and +the shadow bytemap, respectively. The memory object itself is in this case +``&first->info``. Just beware that the start of this dump is NOT the start +of the object itself! The position of the caret (^) corresponds with the +address of the read (ffff88003e4a2024). + +The shadow bytemap dump legend is as follows: + +- i: initialized +- u: uninitialized +- a: unallocated (memory has been allocated by the slab layer, but has not + yet been handed off to anybody) +- f: freed (memory has been allocated by the slab layer, but has been freed + by the previous owner) + +In order to figure out where (relative to the start of the object) the +uninitialized memory was located, we have to look at the disassembly. For +that, we'll need the RIP address again:: + + RIP: 0010:[] [] __dequeue_signal+0xc8/0x190 + + $ objdump -d --no-show-raw-insn vmlinux | grep -C 8 ffffffff8104ede8: + ffffffff8104edc8: mov %r8,0x8(%r8) + ffffffff8104edcc: test %r10d,%r10d + ffffffff8104edcf: js ffffffff8104ee88 <__dequeue_signal+0x168> + ffffffff8104edd5: mov %rax,%rdx + ffffffff8104edd8: mov $0xc,%ecx + ffffffff8104eddd: mov %r13,%rdi + ffffffff8104ede0: mov $0x30,%eax + ffffffff8104ede5: mov %rdx,%rsi + ffffffff8104ede8: rep movsl %ds:(%rsi),%es:(%rdi) + ffffffff8104edea: test $0x2,%al + ffffffff8104edec: je ffffffff8104edf0 <__dequeue_signal+0xd0> + ffffffff8104edee: movsw %ds:(%rsi),%es:(%rdi) + ffffffff8104edf0: test $0x1,%al + ffffffff8104edf2: je ffffffff8104edf5 <__dequeue_signal+0xd5> + ffffffff8104edf4: movsb %ds:(%rsi),%es:(%rdi) + ffffffff8104edf5: mov %r8,%rdi + ffffffff8104edf8: callq ffffffff8104de60 <__sigqueue_free> + +As expected, it's the "``rep movsl``" instruction from the ``memcpy()`` +that causes the warning. We know about ``REP MOVSL`` that it uses the register +``RCX`` to count the number of remaining iterations. By taking a look at the +register dump again (from the kmemcheck report), we can figure out how many +bytes were left to copy:: + + RAX: 0000000000000030 RBX: ffff88003d4ea968 RCX: 0000000000000009 + +By looking at the disassembly, we also see that ``%ecx`` is being loaded +with the value ``$0xc`` just before (ffffffff8104edd8), so we are very +lucky. Keep in mind that this is the number of iterations, not bytes. And +since this is a "long" operation, we need to multiply by 4 to get the +number of bytes. So this means that the uninitialized value was encountered +at 4 * (0xc - 0x9) = 12 bytes from the start of the object. + +We can now try to figure out which field of the "``struct siginfo``" that +was not initialized. This is the beginning of the struct:: + + 40 typedef struct siginfo { + 41 int si_signo; + 42 int si_errno; + 43 int si_code; + 44 + 45 union { + .. + 92 } _sifields; + 93 } siginfo_t; + +On 64-bit, the int is 4 bytes long, so it must the union member that has +not been initialized. We can verify this using gdb:: + + $ gdb vmlinux + ... + (gdb) p &((struct siginfo *) 0)->_sifields + $1 = (union {...} *) 0x10 + +Actually, it seems that the union member is located at offset 0x10 -- which +means that gcc has inserted 4 bytes of padding between the members ``si_code`` +and ``_sifields``. We can now get a fuller picture of the memory dump:: + + _----------------------------=> si_code + / _--------------------=> (padding) + | / _------------=> _sifields(._kill._pid) + | | / _----=> _sifields(._kill._uid) + | | | / + -------|-------|-------|-------| + 80000000000000000000000000000000000000000088ffff0000000000000000 + i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u + +This allows us to realize another important fact: ``si_code`` contains the +value 0x80. Remember that x86 is little endian, so the first 4 bytes +"80000000" are really the number 0x00000080. With a bit of research, we +find that this is actually the constant ``SI_KERNEL`` defined in +``include/asm-generic/siginfo.h``:: + + 144 #define SI_KERNEL 0x80 /* sent by the kernel from somewhere */ + +This macro is used in exactly one place in the x86 kernel: In ``send_signal()`` +in ``kernel/signal.c``:: + + 816 static int send_signal(int sig, struct siginfo *info, struct task_struct *t, + 817 int group) + 818 { + ... + 828 pending = group ? &t->signal->shared_pending : &t->pending; + ... + 851 q = __sigqueue_alloc(t, GFP_ATOMIC, (sig < SIGRTMIN && + 852 (is_si_special(info) || + 853 info->si_code >= 0))); + 854 if (q) { + 855 list_add_tail(&q->list, &pending->list); + 856 switch ((unsigned long) info) { + ... + 865 case (unsigned long) SEND_SIG_PRIV: + 866 q->info.si_signo = sig; + 867 q->info.si_errno = 0; + 868 q->info.si_code = SI_KERNEL; + 869 q->info.si_pid = 0; + 870 q->info.si_uid = 0; + 871 break; + ... + 890 } + +Not only does this match with the ``.si_code`` member, it also matches the place +we found earlier when looking for where siginfo_t objects are enqueued on the +``shared_pending`` list. + +So to sum up: It seems that it is the padding introduced by the compiler +between two struct fields that is uninitialized, and this gets reported when +we do a ``memcpy()`` on the struct. This means that we have identified a false +positive warning. + +Normally, kmemcheck will not report uninitialized accesses in ``memcpy()`` calls +when both the source and destination addresses are tracked. (Instead, we copy +the shadow bytemap as well). In this case, the destination address clearly +was not tracked. We can dig a little deeper into the stack trace from above:: + + arch/x86/kernel/signal.c:805 + arch/x86/kernel/signal.c:871 + arch/x86/kernel/entry_64.S:694 + +And we clearly see that the destination siginfo object is located on the +stack:: + + 782 static void do_signal(struct pt_regs *regs) + 783 { + 784 struct k_sigaction ka; + 785 siginfo_t info; + ... + 804 signr = get_signal_to_deliver(&info, &ka, regs, NULL); + ... + 854 } + +And this ``&info`` is what eventually gets passed to ``copy_siginfo()`` as the +destination argument. + +Now, even though we didn't find an actual error here, the example is still a +good one, because it shows how one would go about to find out what the report +was all about. + + +Annotating false positives +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are a few different ways to make annotations in the source code that +will keep kmemcheck from checking and reporting certain allocations. Here +they are: + +- ``__GFP_NOTRACK_FALSE_POSITIVE`` + This flag can be passed to ``kmalloc()`` or ``kmem_cache_alloc()`` + (therefore also to other functions that end up calling one of + these) to indicate that the allocation should not be tracked + because it would lead to a false positive report. This is a "big + hammer" way of silencing kmemcheck; after all, even if the false + positive pertains to particular field in a struct, for example, we + will now lose the ability to find (real) errors in other parts of + the same struct. + + Example:: + + /* No warnings will ever trigger on accessing any part of x */ + x = kmalloc(sizeof *x, GFP_KERNEL | __GFP_NOTRACK_FALSE_POSITIVE); + +- ``kmemcheck_bitfield_begin(name)``/``kmemcheck_bitfield_end(name)`` and + ``kmemcheck_annotate_bitfield(ptr, name)`` + The first two of these three macros can be used inside struct + definitions to signal, respectively, the beginning and end of a + bitfield. Additionally, this will assign the bitfield a name, which + is given as an argument to the macros. + + Having used these markers, one can later use + kmemcheck_annotate_bitfield() at the point of allocation, to indicate + which parts of the allocation is part of a bitfield. + + Example:: + + struct foo { + int x; + + kmemcheck_bitfield_begin(flags); + int flag_a:1; + int flag_b:1; + kmemcheck_bitfield_end(flags); + + int y; + }; + + struct foo *x = kmalloc(sizeof *x); + + /* No warnings will trigger on accessing the bitfield of x */ + kmemcheck_annotate_bitfield(x, flags); + + Note that ``kmemcheck_annotate_bitfield()`` can be used even before the + return value of ``kmalloc()`` is checked -- in other words, passing NULL + as the first argument is legal (and will do nothing). + + +Reporting errors +---------------- + +As we have seen, kmemcheck will produce false positive reports. Therefore, it +is not very wise to blindly post kmemcheck warnings to mailing lists and +maintainers. Instead, I encourage maintainers and developers to find errors +in their own code. If you get a warning, you can try to work around it, try +to figure out if it's a real error or not, or simply ignore it. Most +developers know their own code and will quickly and efficiently determine the +root cause of a kmemcheck report. This is therefore also the most efficient +way to work with kmemcheck. + +That said, we (the kmemcheck maintainers) will always be on the lookout for +false positives that we can annotate and silence. So whatever you find, +please drop us a note privately! Kernel configs and steps to reproduce (if +available) are of course a great help too. + +Happy hacking! + + +Technical description +--------------------- + +kmemcheck works by marking memory pages non-present. This means that whenever +somebody attempts to access the page, a page fault is generated. The page +fault handler notices that the page was in fact only hidden, and so it calls +on the kmemcheck code to make further investigations. + +When the investigations are completed, kmemcheck "shows" the page by marking +it present (as it would be under normal circumstances). This way, the +interrupted code can continue as usual. + +But after the instruction has been executed, we should hide the page again, so +that we can catch the next access too! Now kmemcheck makes use of a debugging +feature of the processor, namely single-stepping. When the processor has +finished the one instruction that generated the memory access, a debug +exception is raised. From here, we simply hide the page again and continue +execution, this time with the single-stepping feature turned off. + +kmemcheck requires some assistance from the memory allocator in order to work. +The memory allocator needs to + + 1. Tell kmemcheck about newly allocated pages and pages that are about to + be freed. This allows kmemcheck to set up and tear down the shadow memory + for the pages in question. The shadow memory stores the status of each + byte in the allocation proper, e.g. whether it is initialized or + uninitialized. + + 2. Tell kmemcheck which parts of memory should be marked uninitialized. + There are actually a few more states, such as "not yet allocated" and + "recently freed". + +If a slab cache is set up using the SLAB_NOTRACK flag, it will never return +memory that can take page faults because of kmemcheck. + +If a slab cache is NOT set up using the SLAB_NOTRACK flag, callers can still +request memory with the __GFP_NOTRACK or __GFP_NOTRACK_FALSE_POSITIVE flags. +This does not prevent the page faults from occurring, however, but marks the +object in question as being initialized so that no warnings will ever be +produced for this object. + +Currently, the SLAB and SLUB allocators are supported by kmemcheck. diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 3b6382a58301..43f7deeccae4 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -21,3 +21,4 @@ whole; patches welcome! kasan ubsan kmemleak + kmemcheck diff --git a/Documentation/kmemcheck.txt b/Documentation/kmemcheck.txt deleted file mode 100644 index 80aae85d8da6..000000000000 --- a/Documentation/kmemcheck.txt +++ /dev/null @@ -1,754 +0,0 @@ -GETTING STARTED WITH KMEMCHECK -============================== - -Vegard Nossum - - -Contents -======== -0. Introduction -1. Downloading -2. Configuring and compiling -3. How to use -3.1. Booting -3.2. Run-time enable/disable -3.3. Debugging -3.4. Annotating false positives -4. Reporting errors -5. Technical description - - -0. Introduction -=============== - -kmemcheck is a debugging feature for the Linux Kernel. More specifically, it -is a dynamic checker that detects and warns about some uses of uninitialized -memory. - -Userspace programmers might be familiar with Valgrind's memcheck. The main -difference between memcheck and kmemcheck is that memcheck works for userspace -programs only, and kmemcheck works for the kernel only. The implementations -are of course vastly different. Because of this, kmemcheck is not as accurate -as memcheck, but it turns out to be good enough in practice to discover real -programmer errors that the compiler is not able to find through static -analysis. - -Enabling kmemcheck on a kernel will probably slow it down to the extent that -the machine will not be usable for normal workloads such as e.g. an -interactive desktop. kmemcheck will also cause the kernel to use about twice -as much memory as normal. For this reason, kmemcheck is strictly a debugging -feature. - - -1. Downloading -============== - -As of version 2.6.31-rc1, kmemcheck is included in the mainline kernel. - - -2. Configuring and compiling -============================ - -kmemcheck only works for the x86 (both 32- and 64-bit) platform. A number of -configuration variables must have specific settings in order for the kmemcheck -menu to even appear in "menuconfig". These are: - - o CONFIG_CC_OPTIMIZE_FOR_SIZE=n - - This option is located under "General setup" / "Optimize for size". - - Without this, gcc will use certain optimizations that usually lead to - false positive warnings from kmemcheck. An example of this is a 16-bit - field in a struct, where gcc may load 32 bits, then discard the upper - 16 bits. kmemcheck sees only the 32-bit load, and may trigger a - warning for the upper 16 bits (if they're uninitialized). - - o CONFIG_SLAB=y or CONFIG_SLUB=y - - This option is located under "General setup" / "Choose SLAB - allocator". - - o CONFIG_FUNCTION_TRACER=n - - This option is located under "Kernel hacking" / "Tracers" / "Kernel - Function Tracer" - - When function tracing is compiled in, gcc emits a call to another - function at the beginning of every function. This means that when the - page fault handler is called, the ftrace framework will be called - before kmemcheck has had a chance to handle the fault. If ftrace then - modifies memory that was tracked by kmemcheck, the result is an - endless recursive page fault. - - o CONFIG_DEBUG_PAGEALLOC=n - - This option is located under "Kernel hacking" / "Memory Debugging" - / "Debug page memory allocations". - -In addition, I highly recommend turning on CONFIG_DEBUG_INFO=y. This is also -located under "Kernel hacking". With this, you will be able to get line number -information from the kmemcheck warnings, which is extremely valuable in -debugging a problem. This option is not mandatory, however, because it slows -down the compilation process and produces a much bigger kernel image. - -Now the kmemcheck menu should be visible (under "Kernel hacking" / "Memory -Debugging" / "kmemcheck: trap use of uninitialized memory"). Here follows -a description of the kmemcheck configuration variables: - - o CONFIG_KMEMCHECK - - This must be enabled in order to use kmemcheck at all... - - o CONFIG_KMEMCHECK_[DISABLED | ENABLED | ONESHOT]_BY_DEFAULT - - This option controls the status of kmemcheck at boot-time. "Enabled" - will enable kmemcheck right from the start, "disabled" will boot the - kernel as normal (but with the kmemcheck code compiled in, so it can - be enabled at run-time after the kernel has booted), and "one-shot" is - a special mode which will turn kmemcheck off automatically after - detecting the first use of uninitialized memory. - - If you are using kmemcheck to actively debug a problem, then you - probably want to choose "enabled" here. - - The one-shot mode is mostly useful in automated test setups because it - can prevent floods of warnings and increase the chances of the machine - surviving in case something is really wrong. In other cases, the one- - shot mode could actually be counter-productive because it would turn - itself off at the very first error -- in the case of a false positive - too -- and this would come in the way of debugging the specific - problem you were interested in. - - If you would like to use your kernel as normal, but with a chance to - enable kmemcheck in case of some problem, it might be a good idea to - choose "disabled" here. When kmemcheck is disabled, most of the run- - time overhead is not incurred, and the kernel will be almost as fast - as normal. - - o CONFIG_KMEMCHECK_QUEUE_SIZE - - Select the maximum number of error reports to store in an internal - (fixed-size) buffer. Since errors can occur virtually anywhere and in - any context, we need a temporary storage area which is guaranteed not - to generate any other page faults when accessed. The queue will be - emptied as soon as a tasklet may be scheduled. If the queue is full, - new error reports will be lost. - - The default value of 64 is probably fine. If some code produces more - than 64 errors within an irqs-off section, then the code is likely to - produce many, many more, too, and these additional reports seldom give - any more information (the first report is usually the most valuable - anyway). - - This number might have to be adjusted if you are not using serial - console or similar to capture the kernel log. If you are using the - "dmesg" command to save the log, then getting a lot of kmemcheck - warnings might overflow the kernel log itself, and the earlier reports - will get lost in that way instead. Try setting this to 10 or so on - such a setup. - - o CONFIG_KMEMCHECK_SHADOW_COPY_SHIFT - - Select the number of shadow bytes to save along with each entry of the - error-report queue. These bytes indicate what parts of an allocation - are initialized, uninitialized, etc. and will be displayed when an - error is detected to help the debugging of a particular problem. - - The number entered here is actually the logarithm of the number of - bytes that will be saved. So if you pick for example 5 here, kmemcheck - will save 2^5 = 32 bytes. - - The default value should be fine for debugging most problems. It also - fits nicely within 80 columns. - - o CONFIG_KMEMCHECK_PARTIAL_OK - - This option (when enabled) works around certain GCC optimizations that - produce 32-bit reads from 16-bit variables where the upper 16 bits are - thrown away afterwards. - - The default value (enabled) is recommended. This may of course hide - some real errors, but disabling it would probably produce a lot of - false positives. - - o CONFIG_KMEMCHECK_BITOPS_OK - - This option silences warnings that would be generated for bit-field - accesses where not all the bits are initialized at the same time. This - may also hide some real bugs. - - This option is probably obsolete, or it should be replaced with - the kmemcheck-/bitfield-annotations for the code in question. The - default value is therefore fine. - -Now compile the kernel as usual. - - -3. How to use -============= - -3.1. Booting -============ - -First some information about the command-line options. There is only one -option specific to kmemcheck, and this is called "kmemcheck". It can be used -to override the default mode as chosen by the CONFIG_KMEMCHECK_*_BY_DEFAULT -option. Its possible settings are: - - o kmemcheck=0 (disabled) - o kmemcheck=1 (enabled) - o kmemcheck=2 (one-shot mode) - -If SLUB debugging has been enabled in the kernel, it may take precedence over -kmemcheck in such a way that the slab caches which are under SLUB debugging -will not be tracked by kmemcheck. In order to ensure that this doesn't happen -(even though it shouldn't by default), use SLUB's boot option "slub_debug", -like this: slub_debug=- - -In fact, this option may also be used for fine-grained control over SLUB vs. -kmemcheck. For example, if the command line includes "kmemcheck=1 -slub_debug=,dentry", then SLUB debugging will be used only for the "dentry" -slab cache, and with kmemcheck tracking all the other caches. This is advanced -usage, however, and is not generally recommended. - - -3.2. Run-time enable/disable -============================ - -When the kernel has booted, it is possible to enable or disable kmemcheck at -run-time. WARNING: This feature is still experimental and may cause false -positive warnings to appear. Therefore, try not to use this. If you find that -it doesn't work properly (e.g. you see an unreasonable amount of warnings), I -will be happy to take bug reports. - -Use the file /proc/sys/kernel/kmemcheck for this purpose, e.g.: - - $ echo 0 > /proc/sys/kernel/kmemcheck # disables kmemcheck - -The numbers are the same as for the kmemcheck= command-line option. - - -3.3. Debugging -============== - -A typical report will look something like this: - -WARNING: kmemcheck: Caught 32-bit read from uninitialized memory (ffff88003e4a2024) -80000000000000000000000000000000000000000088ffff0000000000000000 - i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u - ^ - -Pid: 1856, comm: ntpdate Not tainted 2.6.29-rc5 #264 945P-A -RIP: 0010:[] [] __dequeue_signal+0xc8/0x190 -RSP: 0018:ffff88003cdf7d98 EFLAGS: 00210002 -RAX: 0000000000000030 RBX: ffff88003d4ea968 RCX: 0000000000000009 -RDX: ffff88003e5d6018 RSI: ffff88003e5d6024 RDI: ffff88003cdf7e84 -RBP: ffff88003cdf7db8 R08: ffff88003e5d6000 R09: 0000000000000000 -R10: 0000000000000080 R11: 0000000000000000 R12: 000000000000000e -R13: ffff88003cdf7e78 R14: ffff88003d530710 R15: ffff88003d5a98c8 -FS: 0000000000000000(0000) GS:ffff880001982000(0063) knlGS:00000 -CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 -CR2: ffff88003f806ea0 CR3: 000000003c036000 CR4: 00000000000006a0 -DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 -DR3: 0000000000000000 DR6: 00000000ffff4ff0 DR7: 0000000000000400 - [] dequeue_signal+0x8e/0x170 - [] get_signal_to_deliver+0x98/0x390 - [] do_notify_resume+0xad/0x7d0 - [] int_signal+0x12/0x17 - [] 0xffffffffffffffff - -The single most valuable information in this report is the RIP (or EIP on 32- -bit) value. This will help us pinpoint exactly which instruction that caused -the warning. - -If your kernel was compiled with CONFIG_DEBUG_INFO=y, then all we have to do -is give this address to the addr2line program, like this: - - $ addr2line -e vmlinux -i ffffffff8104ede8 - arch/x86/include/asm/string_64.h:12 - include/asm-generic/siginfo.h:287 - kernel/signal.c:380 - kernel/signal.c:410 - -The "-e vmlinux" tells addr2line which file to look in. IMPORTANT: This must -be the vmlinux of the kernel that produced the warning in the first place! If -not, the line number information will almost certainly be wrong. - -The "-i" tells addr2line to also print the line numbers of inlined functions. -In this case, the flag was very important, because otherwise, it would only -have printed the first line, which is just a call to memcpy(), which could be -called from a thousand places in the kernel, and is therefore not very useful. -These inlined functions would not show up in the stack trace above, simply -because the kernel doesn't load the extra debugging information. This -technique can of course be used with ordinary kernel oopses as well. - -In this case, it's the caller of memcpy() that is interesting, and it can be -found in include/asm-generic/siginfo.h, line 287: - -281 static inline void copy_siginfo(struct siginfo *to, struct siginfo *from) -282 { -283 if (from->si_code < 0) -284 memcpy(to, from, sizeof(*to)); -285 else -286 /* _sigchld is currently the largest know union member */ -287 memcpy(to, from, __ARCH_SI_PREAMBLE_SIZE + sizeof(from->_sifields._sigchld)); -288 } - -Since this was a read (kmemcheck usually warns about reads only, though it can -warn about writes to unallocated or freed memory as well), it was probably the -"from" argument which contained some uninitialized bytes. Following the chain -of calls, we move upwards to see where "from" was allocated or initialized, -kernel/signal.c, line 380: - -359 static void collect_signal(int sig, struct sigpending *list, siginfo_t *info) -360 { -... -367 list_for_each_entry(q, &list->list, list) { -368 if (q->info.si_signo == sig) { -369 if (first) -370 goto still_pending; -371 first = q; -... -377 if (first) { -378 still_pending: -379 list_del_init(&first->list); -380 copy_siginfo(info, &first->info); -381 __sigqueue_free(first); -... -392 } -393 } - -Here, it is &first->info that is being passed on to copy_siginfo(). The -variable "first" was found on a list -- passed in as the second argument to -collect_signal(). We continue our journey through the stack, to figure out -where the item on "list" was allocated or initialized. We move to line 410: - -395 static int __dequeue_signal(struct sigpending *pending, sigset_t *mask, -396 siginfo_t *info) -397 { -... -410 collect_signal(sig, pending, info); -... -414 } - -Now we need to follow the "pending" pointer, since that is being passed on to -collect_signal() as "list". At this point, we've run out of lines from the -"addr2line" output. Not to worry, we just paste the next addresses from the -kmemcheck stack dump, i.e.: - - [] dequeue_signal+0x8e/0x170 - [] get_signal_to_deliver+0x98/0x390 - [] do_notify_resume+0xad/0x7d0 - [] int_signal+0x12/0x17 - - $ addr2line -e vmlinux -i ffffffff8104f04e ffffffff81050bd8 \ - ffffffff8100b87d ffffffff8100c7b5 - kernel/signal.c:446 - kernel/signal.c:1806 - arch/x86/kernel/signal.c:805 - arch/x86/kernel/signal.c:871 - arch/x86/kernel/entry_64.S:694 - -Remember that since these addresses were found on the stack and not as the -RIP value, they actually point to the _next_ instruction (they are return -addresses). This becomes obvious when we look at the code for line 446: - -422 int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info) -423 { -... -431 signr = __dequeue_signal(&tsk->signal->shared_pending, -432 mask, info); -433 /* -434 * itimer signal ? -435 * -436 * itimers are process shared and we restart periodic -437 * itimers in the signal delivery path to prevent DoS -438 * attacks in the high resolution timer case. This is -439 * compliant with the old way of self restarting -440 * itimers, as the SIGALRM is a legacy signal and only -441 * queued once. Changing the restart behaviour to -442 * restart the timer in the signal dequeue path is -443 * reducing the timer noise on heavy loaded !highres -444 * systems too. -445 */ -446 if (unlikely(signr == SIGALRM)) { -... -489 } - -So instead of looking at 446, we should be looking at 431, which is the line -that executes just before 446. Here we see that what we are looking for is -&tsk->signal->shared_pending. - -Our next task is now to figure out which function that puts items on this -"shared_pending" list. A crude, but efficient tool, is git grep: - - $ git grep -n 'shared_pending' kernel/ - ... - kernel/signal.c:828: pending = group ? &t->signal->shared_pending : &t->pending; - kernel/signal.c:1339: pending = group ? &t->signal->shared_pending : &t->pending; - ... - -There were more results, but none of them were related to list operations, -and these were the only assignments. We inspect the line numbers more closely -and find that this is indeed where items are being added to the list: - -816 static int send_signal(int sig, struct siginfo *info, struct task_struct *t, -817 int group) -818 { -... -828 pending = group ? &t->signal->shared_pending : &t->pending; -... -851 q = __sigqueue_alloc(t, GFP_ATOMIC, (sig < SIGRTMIN && -852 (is_si_special(info) || -853 info->si_code >= 0))); -854 if (q) { -855 list_add_tail(&q->list, &pending->list); -... -890 } - -and: - -1309 int send_sigqueue(struct sigqueue *q, struct task_struct *t, int group) -1310 { -.... -1339 pending = group ? &t->signal->shared_pending : &t->pending; -1340 list_add_tail(&q->list, &pending->list); -.... -1347 } - -In the first case, the list element we are looking for, "q", is being returned -from the function __sigqueue_alloc(), which looks like an allocation function. -Let's take a look at it: - -187 static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags, -188 int override_rlimit) -189 { -190 struct sigqueue *q = NULL; -191 struct user_struct *user; -192 -193 /* -194 * We won't get problems with the target's UID changing under us -195 * because changing it requires RCU be used, and if t != current, the -196 * caller must be holding the RCU readlock (by way of a spinlock) and -197 * we use RCU protection here -198 */ -199 user = get_uid(__task_cred(t)->user); -200 atomic_inc(&user->sigpending); -201 if (override_rlimit || -202 atomic_read(&user->sigpending) <= -203 t->signal->rlim[RLIMIT_SIGPENDING].rlim_cur) -204 q = kmem_cache_alloc(sigqueue_cachep, flags); -205 if (unlikely(q == NULL)) { -206 atomic_dec(&user->sigpending); -207 free_uid(user); -208 } else { -209 INIT_LIST_HEAD(&q->list); -210 q->flags = 0; -211 q->user = user; -212 } -213 -214 return q; -215 } - -We see that this function initializes q->list, q->flags, and q->user. It seems -that now is the time to look at the definition of "struct sigqueue", e.g.: - -14 struct sigqueue { -15 struct list_head list; -16 int flags; -17 siginfo_t info; -18 struct user_struct *user; -19 }; - -And, you might remember, it was a memcpy() on &first->info that caused the -warning, so this makes perfect sense. It also seems reasonable to assume that -it is the caller of __sigqueue_alloc() that has the responsibility of filling -out (initializing) this member. - -But just which fields of the struct were uninitialized? Let's look at -kmemcheck's report again: - -WARNING: kmemcheck: Caught 32-bit read from uninitialized memory (ffff88003e4a2024) -80000000000000000000000000000000000000000088ffff0000000000000000 - i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u - ^ - -These first two lines are the memory dump of the memory object itself, and the -shadow bytemap, respectively. The memory object itself is in this case -&first->info. Just beware that the start of this dump is NOT the start of the -object itself! The position of the caret (^) corresponds with the address of -the read (ffff88003e4a2024). - -The shadow bytemap dump legend is as follows: - - i - initialized - u - uninitialized - a - unallocated (memory has been allocated by the slab layer, but has not - yet been handed off to anybody) - f - freed (memory has been allocated by the slab layer, but has been freed - by the previous owner) - -In order to figure out where (relative to the start of the object) the -uninitialized memory was located, we have to look at the disassembly. For -that, we'll need the RIP address again: - -RIP: 0010:[] [] __dequeue_signal+0xc8/0x190 - - $ objdump -d --no-show-raw-insn vmlinux | grep -C 8 ffffffff8104ede8: - ffffffff8104edc8: mov %r8,0x8(%r8) - ffffffff8104edcc: test %r10d,%r10d - ffffffff8104edcf: js ffffffff8104ee88 <__dequeue_signal+0x168> - ffffffff8104edd5: mov %rax,%rdx - ffffffff8104edd8: mov $0xc,%ecx - ffffffff8104eddd: mov %r13,%rdi - ffffffff8104ede0: mov $0x30,%eax - ffffffff8104ede5: mov %rdx,%rsi - ffffffff8104ede8: rep movsl %ds:(%rsi),%es:(%rdi) - ffffffff8104edea: test $0x2,%al - ffffffff8104edec: je ffffffff8104edf0 <__dequeue_signal+0xd0> - ffffffff8104edee: movsw %ds:(%rsi),%es:(%rdi) - ffffffff8104edf0: test $0x1,%al - ffffffff8104edf2: je ffffffff8104edf5 <__dequeue_signal+0xd5> - ffffffff8104edf4: movsb %ds:(%rsi),%es:(%rdi) - ffffffff8104edf5: mov %r8,%rdi - ffffffff8104edf8: callq ffffffff8104de60 <__sigqueue_free> - -As expected, it's the "rep movsl" instruction from the memcpy() that causes -the warning. We know about REP MOVSL that it uses the register RCX to count -the number of remaining iterations. By taking a look at the register dump -again (from the kmemcheck report), we can figure out how many bytes were left -to copy: - -RAX: 0000000000000030 RBX: ffff88003d4ea968 RCX: 0000000000000009 - -By looking at the disassembly, we also see that %ecx is being loaded with the -value $0xc just before (ffffffff8104edd8), so we are very lucky. Keep in mind -that this is the number of iterations, not bytes. And since this is a "long" -operation, we need to multiply by 4 to get the number of bytes. So this means -that the uninitialized value was encountered at 4 * (0xc - 0x9) = 12 bytes -from the start of the object. - -We can now try to figure out which field of the "struct siginfo" that was not -initialized. This is the beginning of the struct: - -40 typedef struct siginfo { -41 int si_signo; -42 int si_errno; -43 int si_code; -44 -45 union { -.. -92 } _sifields; -93 } siginfo_t; - -On 64-bit, the int is 4 bytes long, so it must the union member that has -not been initialized. We can verify this using gdb: - - $ gdb vmlinux - ... - (gdb) p &((struct siginfo *) 0)->_sifields - $1 = (union {...} *) 0x10 - -Actually, it seems that the union member is located at offset 0x10 -- which -means that gcc has inserted 4 bytes of padding between the members si_code -and _sifields. We can now get a fuller picture of the memory dump: - - _----------------------------=> si_code - / _--------------------=> (padding) - | / _------------=> _sifields(._kill._pid) - | | / _----=> _sifields(._kill._uid) - | | | / --------|-------|-------|-------| -80000000000000000000000000000000000000000088ffff0000000000000000 - i i i i u u u u i i i i i i i i u u u u u u u u u u u u u u u u - -This allows us to realize another important fact: si_code contains the value -0x80. Remember that x86 is little endian, so the first 4 bytes "80000000" are -really the number 0x00000080. With a bit of research, we find that this is -actually the constant SI_KERNEL defined in include/asm-generic/siginfo.h: - -144 #define SI_KERNEL 0x80 /* sent by the kernel from somewhere */ - -This macro is used in exactly one place in the x86 kernel: In send_signal() -in kernel/signal.c: - -816 static int send_signal(int sig, struct siginfo *info, struct task_struct *t, -817 int group) -818 { -... -828 pending = group ? &t->signal->shared_pending : &t->pending; -... -851 q = __sigqueue_alloc(t, GFP_ATOMIC, (sig < SIGRTMIN && -852 (is_si_special(info) || -853 info->si_code >= 0))); -854 if (q) { -855 list_add_tail(&q->list, &pending->list); -856 switch ((unsigned long) info) { -... -865 case (unsigned long) SEND_SIG_PRIV: -866 q->info.si_signo = sig; -867 q->info.si_errno = 0; -868 q->info.si_code = SI_KERNEL; -869 q->info.si_pid = 0; -870 q->info.si_uid = 0; -871 break; -... -890 } - -Not only does this match with the .si_code member, it also matches the place -we found earlier when looking for where siginfo_t objects are enqueued on the -"shared_pending" list. - -So to sum up: It seems that it is the padding introduced by the compiler -between two struct fields that is uninitialized, and this gets reported when -we do a memcpy() on the struct. This means that we have identified a false -positive warning. - -Normally, kmemcheck will not report uninitialized accesses in memcpy() calls -when both the source and destination addresses are tracked. (Instead, we copy -the shadow bytemap as well). In this case, the destination address clearly -was not tracked. We can dig a little deeper into the stack trace from above: - - arch/x86/kernel/signal.c:805 - arch/x86/kernel/signal.c:871 - arch/x86/kernel/entry_64.S:694 - -And we clearly see that the destination siginfo object is located on the -stack: - -782 static void do_signal(struct pt_regs *regs) -783 { -784 struct k_sigaction ka; -785 siginfo_t info; -... -804 signr = get_signal_to_deliver(&info, &ka, regs, NULL); -... -854 } - -And this &info is what eventually gets passed to copy_siginfo() as the -destination argument. - -Now, even though we didn't find an actual error here, the example is still a -good one, because it shows how one would go about to find out what the report -was all about. - - -3.4. Annotating false positives -=============================== - -There are a few different ways to make annotations in the source code that -will keep kmemcheck from checking and reporting certain allocations. Here -they are: - - o __GFP_NOTRACK_FALSE_POSITIVE - - This flag can be passed to kmalloc() or kmem_cache_alloc() (therefore - also to other functions that end up calling one of these) to indicate - that the allocation should not be tracked because it would lead to - a false positive report. This is a "big hammer" way of silencing - kmemcheck; after all, even if the false positive pertains to - particular field in a struct, for example, we will now lose the - ability to find (real) errors in other parts of the same struct. - - Example: - - /* No warnings will ever trigger on accessing any part of x */ - x = kmalloc(sizeof *x, GFP_KERNEL | __GFP_NOTRACK_FALSE_POSITIVE); - - o kmemcheck_bitfield_begin(name)/kmemcheck_bitfield_end(name) and - kmemcheck_annotate_bitfield(ptr, name) - - The first two of these three macros can be used inside struct - definitions to signal, respectively, the beginning and end of a - bitfield. Additionally, this will assign the bitfield a name, which - is given as an argument to the macros. - - Having used these markers, one can later use - kmemcheck_annotate_bitfield() at the point of allocation, to indicate - which parts of the allocation is part of a bitfield. - - Example: - - struct foo { - int x; - - kmemcheck_bitfield_begin(flags); - int flag_a:1; - int flag_b:1; - kmemcheck_bitfield_end(flags); - - int y; - }; - - struct foo *x = kmalloc(sizeof *x); - - /* No warnings will trigger on accessing the bitfield of x */ - kmemcheck_annotate_bitfield(x, flags); - - Note that kmemcheck_annotate_bitfield() can be used even before the - return value of kmalloc() is checked -- in other words, passing NULL - as the first argument is legal (and will do nothing). - - -4. Reporting errors -=================== - -As we have seen, kmemcheck will produce false positive reports. Therefore, it -is not very wise to blindly post kmemcheck warnings to mailing lists and -maintainers. Instead, I encourage maintainers and developers to find errors -in their own code. If you get a warning, you can try to work around it, try -to figure out if it's a real error or not, or simply ignore it. Most -developers know their own code and will quickly and efficiently determine the -root cause of a kmemcheck report. This is therefore also the most efficient -way to work with kmemcheck. - -That said, we (the kmemcheck maintainers) will always be on the lookout for -false positives that we can annotate and silence. So whatever you find, -please drop us a note privately! Kernel configs and steps to reproduce (if -available) are of course a great help too. - -Happy hacking! - - -5. Technical description -======================== - -kmemcheck works by marking memory pages non-present. This means that whenever -somebody attempts to access the page, a page fault is generated. The page -fault handler notices that the page was in fact only hidden, and so it calls -on the kmemcheck code to make further investigations. - -When the investigations are completed, kmemcheck "shows" the page by marking -it present (as it would be under normal circumstances). This way, the -interrupted code can continue as usual. - -But after the instruction has been executed, we should hide the page again, so -that we can catch the next access too! Now kmemcheck makes use of a debugging -feature of the processor, namely single-stepping. When the processor has -finished the one instruction that generated the memory access, a debug -exception is raised. From here, we simply hide the page again and continue -execution, this time with the single-stepping feature turned off. - -kmemcheck requires some assistance from the memory allocator in order to work. -The memory allocator needs to - - 1. Tell kmemcheck about newly allocated pages and pages that are about to - be freed. This allows kmemcheck to set up and tear down the shadow memory - for the pages in question. The shadow memory stores the status of each - byte in the allocation proper, e.g. whether it is initialized or - uninitialized. - - 2. Tell kmemcheck which parts of memory should be marked uninitialized. - There are actually a few more states, such as "not yet allocated" and - "recently freed". - -If a slab cache is set up using the SLAB_NOTRACK flag, it will never return -memory that can take page faults because of kmemcheck. - -If a slab cache is NOT set up using the SLAB_NOTRACK flag, callers can still -request memory with the __GFP_NOTRACK or __GFP_NOTRACK_FALSE_POSITIVE flags. -This does not prevent the page faults from occurring, however, but marks the -object in question as being initialized so that no warnings will ever be -produced for this object. - -Currently, the SLAB and SLUB allocators are supported by kmemcheck. diff --git a/MAINTAINERS b/MAINTAINERS index b235e0d53449..810723537aa5 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6803,7 +6803,7 @@ KMEMCHECK M: Vegard Nossum M: Pekka Enberg S: Maintained -F: Documentation/kmemcheck.txt +F: Documentation/dev-tools/kmemcheck.rst F: arch/x86/include/asm/kmemcheck.h F: arch/x86/mm/kmemcheck/ F: include/linux/kmemcheck.h From 5f0962748d46c63aaf5c46dcb1c8f52dfb7b717f Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Mon, 8 Aug 2016 15:55:49 -0600 Subject: [PATCH 41/51] docs: Sphinxify gdb-kernel-debugging.txt and move to dev-tools Acked-by: Jan Kiszka Signed-off-by: Jonathan Corbet --- .../gdb-kernel-debugging.rst} | 77 +++++++++++-------- Documentation/dev-tools/tools.rst | 1 + 2 files changed, 46 insertions(+), 32 deletions(-) rename Documentation/{gdb-kernel-debugging.txt => dev-tools/gdb-kernel-debugging.rst} (73%) diff --git a/Documentation/gdb-kernel-debugging.txt b/Documentation/dev-tools/gdb-kernel-debugging.rst similarity index 73% rename from Documentation/gdb-kernel-debugging.txt rename to Documentation/dev-tools/gdb-kernel-debugging.rst index 7050ce8794b9..5e93c9bc6619 100644 --- a/Documentation/gdb-kernel-debugging.txt +++ b/Documentation/dev-tools/gdb-kernel-debugging.rst @@ -1,3 +1,5 @@ +.. highlight:: none + Debugging kernel and modules via gdb ==================================== @@ -13,54 +15,58 @@ be transferred to the other gdb stubs as well. Requirements ------------ - o gdb 7.2+ (recommended: 7.4+) with python support enabled (typically true - for distributions) +- gdb 7.2+ (recommended: 7.4+) with python support enabled (typically true + for distributions) Setup ----- - o Create a virtual Linux machine for QEMU/KVM (see www.linux-kvm.org and - www.qemu.org for more details). For cross-development, - http://landley.net/aboriginal/bin keeps a pool of machine images and - toolchains that can be helpful to start from. +- Create a virtual Linux machine for QEMU/KVM (see www.linux-kvm.org and + www.qemu.org for more details). For cross-development, + http://landley.net/aboriginal/bin keeps a pool of machine images and + toolchains that can be helpful to start from. - o Build the kernel with CONFIG_GDB_SCRIPTS enabled, but leave - CONFIG_DEBUG_INFO_REDUCED off. If your architecture supports - CONFIG_FRAME_POINTER, keep it enabled. +- Build the kernel with CONFIG_GDB_SCRIPTS enabled, but leave + CONFIG_DEBUG_INFO_REDUCED off. If your architecture supports + CONFIG_FRAME_POINTER, keep it enabled. - o Install that kernel on the guest. +- Install that kernel on the guest. + Alternatively, QEMU allows to boot the kernel directly using -kernel, + -append, -initrd command line switches. This is generally only useful if + you do not depend on modules. See QEMU documentation for more details on + this mode. - Alternatively, QEMU allows to boot the kernel directly using -kernel, - -append, -initrd command line switches. This is generally only useful if - you do not depend on modules. See QEMU documentation for more details on - this mode. +- Enable the gdb stub of QEMU/KVM, either - o Enable the gdb stub of QEMU/KVM, either - at VM startup time by appending "-s" to the QEMU command line - or + + or + - during runtime by issuing "gdbserver" from the QEMU monitor console - o cd /path/to/linux-build +- cd /path/to/linux-build - o Start gdb: gdb vmlinux +- Start gdb: gdb vmlinux - Note: Some distros may restrict auto-loading of gdb scripts to known safe - directories. In case gdb reports to refuse loading vmlinux-gdb.py, add + Note: Some distros may restrict auto-loading of gdb scripts to known safe + directories. In case gdb reports to refuse loading vmlinux-gdb.py, add:: add-auto-load-safe-path /path/to/linux-build - to ~/.gdbinit. See gdb help for more details. + to ~/.gdbinit. See gdb help for more details. + +- Attach to the booted guest:: - o Attach to the booted guest: (gdb) target remote :1234 Examples of using the Linux-provided gdb helpers ------------------------------------------------ - o Load module (and main kernel) symbols: +- Load module (and main kernel) symbols:: + (gdb) lx-symbols loading vmlinux scanning for modules in /home/user/linux/build @@ -72,17 +78,20 @@ Examples of using the Linux-provided gdb helpers ... loading @0xffffffffa0000000: /home/user/linux/build/drivers/ata/ata_generic.ko - o Set a breakpoint on some not yet loaded module function, e.g.: +- Set a breakpoint on some not yet loaded module function, e.g.:: + (gdb) b btrfs_init_sysfs Function "btrfs_init_sysfs" not defined. Make breakpoint pending on future shared library load? (y or [n]) y Breakpoint 1 (btrfs_init_sysfs) pending. - o Continue the target +- Continue the target:: + (gdb) c - o Load the module on the target and watch the symbols being loaded as well as - the breakpoint hit: +- Load the module on the target and watch the symbols being loaded as well as + the breakpoint hit:: + loading @0xffffffffa0034000: /home/user/linux/build/lib/libcrc32c.ko loading @0xffffffffa0050000: /home/user/linux/build/lib/lzo/lzo_compress.ko loading @0xffffffffa006e000: /home/user/linux/build/lib/zlib_deflate/zlib_deflate.ko @@ -91,7 +100,8 @@ Examples of using the Linux-provided gdb helpers Breakpoint 1, btrfs_init_sysfs () at /home/user/linux/fs/btrfs/sysfs.c:36 36 btrfs_kset = kset_create_and_add("btrfs", NULL, fs_kobj); - o Dump the log buffer of the target kernel: +- Dump the log buffer of the target kernel:: + (gdb) lx-dmesg [ 0.000000] Initializing cgroup subsys cpuset [ 0.000000] Initializing cgroup subsys cpu @@ -102,19 +112,22 @@ Examples of using the Linux-provided gdb helpers [ 0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved .... - o Examine fields of the current task struct: +- Examine fields of the current task struct:: + (gdb) p $lx_current().pid $1 = 4998 (gdb) p $lx_current().comm $2 = "modprobe\000\000\000\000\000\000\000" - o Make use of the per-cpu function for the current or a specified CPU: +- Make use of the per-cpu function for the current or a specified CPU:: + (gdb) p $lx_per_cpu("runqueues").nr_running $3 = 1 (gdb) p $lx_per_cpu("runqueues", 2).nr_running $4 = 0 - o Dig into hrtimers using the container_of helper: +- Dig into hrtimers using the container_of helper:: + (gdb) set $next = $lx_per_cpu("hrtimer_bases").clock_base[0].active.next (gdb) p *$container_of($next, "struct hrtimer", "node") $5 = { @@ -144,7 +157,7 @@ List of commands and functions ------------------------------ The number of commands and convenience functions may evolve over the time, -this is just a snapshot of the initial version: +this is just a snapshot of the initial version:: (gdb) apropos lx function lx_current -- Return current task diff --git a/Documentation/dev-tools/tools.rst b/Documentation/dev-tools/tools.rst index 43f7deeccae4..824ae8e54dd5 100644 --- a/Documentation/dev-tools/tools.rst +++ b/Documentation/dev-tools/tools.rst @@ -22,3 +22,4 @@ whole; patches welcome! ubsan kmemleak kmemcheck + gdb-kernel-debugging From e8f5c617f26626ef4915ffa176f4ae02c9e08531 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 22 Aug 2016 15:16:21 -0600 Subject: [PATCH 42/51] doc-rst: add boilerplate to customize c-domain Add a sphinx-extension to customize the sphinx c-domain. No functional changes right yet, just the boilerplate code. Signed-off-by: Markus Heiser [ jc: coding-style tweak ] Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 2 +- Documentation/sphinx/cdomain.py | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 Documentation/sphinx/cdomain.py diff --git a/Documentation/conf.py b/Documentation/conf.py index 23e2f0bbcfc8..88c377d468d0 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -34,7 +34,7 @@ from load_config import loadConfig # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['kernel-doc', 'rstFlatTable', 'kernel_include'] +extensions = ['kernel-doc', 'rstFlatTable', 'kernel_include', 'cdomain'] # The name of the math extension changed on Sphinx 1.4 if minor > 3: diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py new file mode 100644 index 000000000000..d6e66e289808 --- /dev/null +++ b/Documentation/sphinx/cdomain.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8; mode: python -*- +u""" + cdomain + ~~~~~~~ + + Replacement for the sphinx c-domain. + + :copyright: Copyright (C) 2016 Markus Heiser + :license: GPL Version 2, June 1991 see Linux/COPYING for details. +""" + +from sphinx.domains.c import CObject as Base_CObject +from sphinx.domains.c import CDomain as Base_CDomain + +__version__ = '1.0' + +def setup(app): + + app.override_domain(CDomain) + + return dict( + version = __version__, + parallel_read_safe = True, + parallel_write_safe = True + ) + +class CObject(Base_CObject): + + """ + Description of a C language object. + """ + +class CDomain(Base_CDomain): + + """C language domain.""" + name = 'c' + label = 'C' + directives = { + 'function': CObject, + 'member': CObject, + 'macro': CObject, + 'type': CObject, + 'var': CObject, + } From 2c645cd7c4a0d4b35da1e43ec3a5b55a64038157 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 15 Aug 2016 16:08:25 +0200 Subject: [PATCH 43/51] doc-rst:c-domain: ref-name of a function declaration Add option 'name' to the "c:function:" directive. With option 'name' the ref-name of a function can be modified. E.g.:: .. c:function:: int ioctl( int fd, int request ) :name: VIDIOC_LOG_STATUS The func-name (e.g. ioctl) remains in the output but the ref-name changed from ``ioctl`` to ``VIDIOC_LOG_STATUS``. The index entry for this function is also changed to ``VIDIOC_LOG_STATUS`` and the function can now referenced by:: :c:func:`VIDIOC_LOG_STATUS` Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/kernel-documentation.rst | 29 ++++++++++++++++++++++++ Documentation/sphinx/cdomain.py | 31 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/Documentation/kernel-documentation.rst b/Documentation/kernel-documentation.rst index c4eb5049da39..bc4c3f526816 100644 --- a/Documentation/kernel-documentation.rst +++ b/Documentation/kernel-documentation.rst @@ -107,6 +107,35 @@ Here are some specific guidelines for the kernel documentation: the order as encountered."), having the higher levels the same overall makes it easier to follow the documents. + +the C domain +------------ + +The `Sphinx C Domain`_ (name c) is suited for documentation of C API. E.g. a +function prototype: + +.. code-block:: rst + + .. c:function:: int ioctl( int fd, int request ) + +The C domain of the kernel-doc has some additional features. E.g. you can +*rename* the reference name of a function with a common name like ``open`` or +``ioctl``: + +.. code-block:: rst + + .. c:function:: int ioctl( int fd, int request ) + :name: VIDIOC_LOG_STATUS + +The func-name (e.g. ioctl) remains in the output but the ref-name changed from +``ioctl`` to ``VIDIOC_LOG_STATUS``. The index entry for this function is also +changed to ``VIDIOC_LOG_STATUS`` and the function can now referenced by: + +.. code-block:: rst + + :c:func:`VIDIOC_LOG_STATUS` + + list tables ----------- diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index d6e66e289808..b1912c80ff77 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -7,8 +7,24 @@ u""" :copyright: Copyright (C) 2016 Markus Heiser :license: GPL Version 2, June 1991 see Linux/COPYING for details. + + List of customizations: + + * Add option 'name' to the "c:function:" directive. With option 'name' the + ref-name of a function can be modified. E.g.:: + + .. c:function:: int ioctl( int fd, int request ) + :name: VIDIOC_LOG_STATUS + + The func-name (e.g. ioctl) remains in the output but the ref-name changed + from 'ioctl' to 'VIDIOC_LOG_STATUS'. The function is referenced by:: + + * :c:func:`VIDIOC_LOG_STATUS` or + * :any:`VIDIOC_LOG_STATUS` (``:any:`` needs sphinx 1.3) """ +from docutils.parsers.rst import directives + from sphinx.domains.c import CObject as Base_CObject from sphinx.domains.c import CDomain as Base_CDomain @@ -29,6 +45,21 @@ class CObject(Base_CObject): """ Description of a C language object. """ + option_spec = { + "name" : directives.unchanged + } + + def handle_signature(self, sig, signode): + """Transform a C signature into RST nodes.""" + fullname = super(CObject, self).handle_signature(sig, signode) + if "name" in self.options: + if self.objtype == 'function': + fullname = self.options["name"] + else: + # FIXME: handle :name: value of other declaration types? + pass + return fullname + class CDomain(Base_CDomain): From 556aa6d5d9616ccfc0099c40dc239157f50ee776 Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 15 Aug 2016 16:08:26 +0200 Subject: [PATCH 44/51] doc-rst: moved *duplicate* warnings to nitpicky mode Moved the *duplicate C object description* warnings for function declarations in the nitpicky mode. In nitpick mode, you can suppress those warnings (e.g. ioctl) with:: nitpicky = True nitpick_ignore = [ ("c:func", "ioctl"), ] See Sphinx documentation for the config values for ``nitpick`` and ``nitpick_ignore`` [1]. With this change all the ".. cpp:function:: int ioctl(..)" descriptions (found in the media book) can be migrated to ".. c:function:: int ioctl(..)", without getting any warnings. E.g.:: .. cpp:function:: int ioctl( int fd, int request, struct cec_event *argp ) .. c:function:: int ioctl( int fd, int request, struct cec_event *argp ) The main effect, is that we get those *CPP-types* back into Sphinx's C- namespace and we need no longer to distinguish between c/cpp references, when we refer a function like the ioctl. [1] http://www.sphinx-doc.org/en/stable/config.html?highlight=nitpick#confval-nitpicky Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/sphinx/cdomain.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index b1912c80ff77..9eb714ada394 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -10,6 +10,10 @@ u""" List of customizations: + * Moved the *duplicate C object description* warnings for function + declarations in the nitpicky mode. See Sphinx documentation for + the config values for ``nitpick`` and ``nitpick_ignore``. + * Add option 'name' to the "c:function:" directive. With option 'name' the ref-name of a function can be modified. E.g.:: @@ -60,6 +64,29 @@ class CObject(Base_CObject): pass return fullname + def add_target_and_index(self, name, sig, signode): + # for C API items we add a prefix since names are usually not qualified + # by a module name and so easily clash with e.g. section titles + targetname = 'c.' + name + if targetname not in self.state.document.ids: + signode['names'].append(targetname) + signode['ids'].append(targetname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + inv = self.env.domaindata['c']['objects'] + if (name in inv and self.env.config.nitpicky): + if self.objtype == 'function': + if ('c:func', name) not in self.env.config.nitpick_ignore: + self.state_machine.reporter.warning( + 'duplicate C object description of %s, ' % name + + 'other instance in ' + self.env.doc2path(inv[name][0]), + line=self.lineno) + inv[name] = (self.env.docname, self.objtype) + + indextext = self.get_index_text(name) + if indextext: + self.indexnode['entries'].append(('single', indextext, + targetname, '', None)) class CDomain(Base_CDomain): From aa10a7826646c56eb4553df8fe81b3d23655c91a Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 15 Aug 2016 16:08:27 +0200 Subject: [PATCH 45/51] doc-rst: Revert "kernel-doc: fix handling of address_space tags" This reverts commit a88b1672d4ddf9895eb53e6980926d5e960dea8e. From the origin comit log:: The RST cpp:function handler is very pedantic: it doesn't allow any macros like __user on it Since the kernel-doc parser does NOT make use of the cpp:domain, there is no need to change the kernel-doc parser eleminating the address_space tags. Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 4f2e9049e8fa..ba081c7636a2 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1849,9 +1849,6 @@ sub output_function_rst(%) { $count++; $type = $args{'parametertypes'}{$parameter}; - # RST doesn't like address_space tags at function prototypes - $type =~ s/__(user|kernel|iomem|percpu|pmem|rcu)\s*//; - if ($type =~ m/([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)/) { # pointer-to-function print $1 . $parameter . ") (" . $2; From 9cd3476c91708b4b814f17671597a2708ec195ed Mon Sep 17 00:00:00 2001 From: Markus Heiser Date: Mon, 15 Aug 2016 16:08:28 +0200 Subject: [PATCH 46/51] doc-rst: migrate ioctl CEC_DQEVENT to c-domain This is only one example, demonstrating the benefits of the patch series. The CEC_DQEVENT ioctl is migrated to the sphinx c-domain and referred by ":name: CEC_DQEVENT". With this change the indirection using ":ref:`CEC_DQEVENT` is no longer needed, we can refer the ioctl directly with ":c:func:`CEC_DQEVENT`". As addition in the index, there is a entry "CEC_DQEVENT (C function)". Signed-off-by: Markus Heiser Signed-off-by: Jonathan Corbet --- Documentation/media/uapi/cec/cec-func-open.rst | 2 +- Documentation/media/uapi/cec/cec-ioc-dqevent.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Documentation/media/uapi/cec/cec-func-open.rst b/Documentation/media/uapi/cec/cec-func-open.rst index 38fd7e0cfccd..7c0f981a6e07 100644 --- a/Documentation/media/uapi/cec/cec-func-open.rst +++ b/Documentation/media/uapi/cec/cec-func-open.rst @@ -32,7 +32,7 @@ Arguments Open flags. Access mode must be ``O_RDWR``. When the ``O_NONBLOCK`` flag is given, the - :ref:`CEC_RECEIVE ` and :ref:`CEC_DQEVENT ` ioctls + :ref:`CEC_RECEIVE ` and :c:func:`CEC_DQEVENT` ioctls will return the ``EAGAIN`` error code when no message or event is available, and ioctls :ref:`CEC_TRANSMIT `, :ref:`CEC_ADAP_S_PHYS_ADDR ` and diff --git a/Documentation/media/uapi/cec/cec-ioc-dqevent.rst b/Documentation/media/uapi/cec/cec-ioc-dqevent.rst index 7a6d6d00ce19..4e12e6cd88ee 100644 --- a/Documentation/media/uapi/cec/cec-ioc-dqevent.rst +++ b/Documentation/media/uapi/cec/cec-ioc-dqevent.rst @@ -15,7 +15,8 @@ CEC_DQEVENT - Dequeue a CEC event Synopsis ======== -.. cpp:function:: int ioctl( int fd, int request, struct cec_event *argp ) +.. c:function:: int ioctl( int fd, int request, struct cec_event *argp ) + :name: CEC_DQEVENT Arguments ========= @@ -36,7 +37,7 @@ Description and is currently only available as a staging kernel module. CEC devices can send asynchronous events. These can be retrieved by -calling :ref:`ioctl CEC_DQEVENT `. If the file descriptor is in +calling :c:func:`CEC_DQEVENT`. If the file descriptor is in non-blocking mode and no event is pending, then it will return -1 and set errno to the ``EAGAIN`` error code. From d565127d120e9b95ba98549c31eab9cec1cbbbc7 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Thu, 18 Aug 2016 11:53:39 -0300 Subject: [PATCH 47/51] docs-rst: add support for LaTeX output Sphinx supports LaTeX output. Sometimes, it is interesting to call it directly, instead of also generating a PDF. As it comes for free, add a target for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/DocBook/Makefile | 1 + Documentation/Makefile.sphinx | 7 ++++++- Makefile | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile index a91c96522379..a558dfcc9e2d 100644 --- a/Documentation/DocBook/Makefile +++ b/Documentation/DocBook/Makefile @@ -72,6 +72,7 @@ installmandocs: mandocs # no-op for the DocBook toolchain epubdocs: +latexdocs: ### #External programs used diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index ba4efb1f68f3..894cfaa41f55 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -66,12 +66,16 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4); htmldocs: @$(foreach var,$(SPHINXDIRS),$(call loop_cmd,sphinx,html,$(var),,$(var))) -pdfdocs: +latexdocs: ifeq ($(HAVE_PDFLATEX),0) $(warning The 'xelatex' command was not found. Make sure you have it installed and in PATH to produce PDF output.) @echo " SKIP Sphinx $@ target." else # HAVE_PDFLATEX @$(call loop_cmd,sphinx,latex,.,latex,.) +endif # HAVE_PDFLATEX + +pdfdocs: latexdocs +ifneq ($(HAVE_PDFLATEX),0) $(Q)$(MAKE) PDFLATEX=xelatex LATEXOPTS="-interaction=nonstopmode" -C $(BUILDDIR)/latex endif # HAVE_PDFLATEX @@ -95,6 +99,7 @@ endif # HAVE_SPHINX dochelp: @echo ' Linux kernel internal documentation in different formats (Sphinx):' @echo ' htmldocs - HTML' + @echo ' latexdocs - LaTeX' @echo ' pdfdocs - PDF' @echo ' epubdocs - EPUB' @echo ' xmldocs - XML' diff --git a/Makefile b/Makefile index 70de1448c571..0fa3feb6f74e 100644 --- a/Makefile +++ b/Makefile @@ -1432,7 +1432,7 @@ $(help-board-dirs): help-%: # Documentation targets # --------------------------------------------------------------------------- -DOC_TARGETS := xmldocs sgmldocs psdocs pdfdocs htmldocs mandocs installmandocs epubdocs cleandocs +DOC_TARGETS := xmldocs sgmldocs psdocs latexdocs pdfdocs htmldocs mandocs installmandocs epubdocs cleandocs PHONY += $(DOC_TARGETS) $(DOC_TARGETS): scripts_basic FORCE $(Q)$(MAKE) $(build)=scripts build_docproc build_check-lc_ctype From aa4e37a3d13679dccf7945dd864375b698cf0df9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 19 Aug 2016 09:49:38 -0300 Subject: [PATCH 48/51] docs-rst: conf.py: adjust the size of .. note:: tag While the current implementation works well when using as a paragraph, it doesn't work properly if inside a table. As we have quite a few such cases, fix the logic to take the column size into account. PS.: I took the logic there from the latest version of Sphinx.sty Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 88c377d468d0..aaa3f70aafcb 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -282,7 +282,13 @@ latex_elements = { \\definecolor{MyGray}{rgb}{0.80,0.80,0.80} \\makeatletter\\newenvironment{graybox}{% - \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\columnwidth}}{\\end{minipage}\\end{lrbox}% + \\newlength{\\py@noticelength} + \\setlength{\\fboxrule}{1pt} + \\setlength{\\fboxsep}{7pt} + \\setlength{\\py@noticelength}{\\linewidth} + \\addtolength{\\py@noticelength}{-2\\fboxsep} + \\addtolength{\\py@noticelength}{-2\\fboxrule} + \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\py@noticelength}}{\\end{minipage}\\end{lrbox}% \\colorbox{MyGray}{\\usebox{\\@tempboxa}} }\\makeatother From 41cff161fe99d1c6a773becc2250a1dc3ac035ff Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 21 Aug 2016 15:23:03 -0300 Subject: [PATCH 49/51] docs-rst: Use better colors for note/warning/attention boxes Instead of painting the box with gray, let's use a colored box. IMHO, that makes easier to warn users about some issue pointed by the Sphinx. It also matches to what we do already with the HTML output. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index aaa3f70aafcb..2a02a817b1a0 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -277,11 +277,12 @@ latex_elements = { % Allow generate some pages in landscape \\usepackage{lscape} - % Put notes in gray color and let them be inside a table - - \\definecolor{MyGray}{rgb}{0.80,0.80,0.80} - - \\makeatletter\\newenvironment{graybox}{% + % Put notes in color and let them be inside a table + \\definecolor{NoteColor}{RGB}{204,255,255} + \\definecolor{WarningColor}{RGB}{255,204,204} + \\definecolor{AttentionColor}{RGB}{255,255,204} + \\definecolor{OtherColor}{RGB}{204,204,204} + \\makeatletter\\newenvironment{coloredbox}[1]{% \\newlength{\\py@noticelength} \\setlength{\\fboxrule}{1pt} \\setlength{\\fboxsep}{7pt} @@ -289,20 +290,33 @@ latex_elements = { \\addtolength{\\py@noticelength}{-2\\fboxsep} \\addtolength{\\py@noticelength}{-2\\fboxrule} \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\py@noticelength}}{\\end{minipage}\\end{lrbox}% - \\colorbox{MyGray}{\\usebox{\\@tempboxa}} + \\ifthenelse% + {\\equal{\\py@noticetype}{note}}% + {\\colorbox{NoteColor}{\\usebox{\\@tempboxa}}}% + {% + \\ifthenelse% + {\\equal{\\py@noticetype}{warning}}% + {\\colorbox{WarningColor}{\\usebox{\\@tempboxa}}}% + {% + \\ifthenelse% + {\\equal{\\py@noticetype}{attention}}% + {\\colorbox{AttentionColor}{\\usebox{\\@tempboxa}}}% + {\\colorbox{OtherColor}{\\usebox{\\@tempboxa}}}% + }% + }% }\\makeatother \\makeatletter - \\renewenvironment{notice}[2]{ - \\begin{graybox} - \\bf\\it + \\renewenvironment{notice}[2]{% \\def\\py@noticetype{#1} + \\begin{coloredbox}{#1} + \\bf\\it \\par\\strong{#2} \\csname py@noticestart@#1\\endcsname } { \\csname py@noticeend@\\py@noticetype\\endcsname - \\end{graybox} + \\end{coloredbox} } \\makeatother From ba1377fba7047a86c31981363cbd9e65b1ca2763 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Sun, 21 Aug 2016 15:23:04 -0300 Subject: [PATCH 50/51] docs-rst: Fix an warning when in interactive mode When XeLaTeX is in interactive mode, it complains that py@noticelength already exists. Rename it and declare it only once to avoid such messages. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index 2a02a817b1a0..85e9f16b1d98 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -282,14 +282,14 @@ latex_elements = { \\definecolor{WarningColor}{RGB}{255,204,204} \\definecolor{AttentionColor}{RGB}{255,255,204} \\definecolor{OtherColor}{RGB}{204,204,204} + \\newlength{\\mynoticelength} \\makeatletter\\newenvironment{coloredbox}[1]{% - \\newlength{\\py@noticelength} \\setlength{\\fboxrule}{1pt} \\setlength{\\fboxsep}{7pt} - \\setlength{\\py@noticelength}{\\linewidth} - \\addtolength{\\py@noticelength}{-2\\fboxsep} - \\addtolength{\\py@noticelength}{-2\\fboxrule} - \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\py@noticelength}}{\\end{minipage}\\end{lrbox}% + \\setlength{\\mynoticelength}{\\linewidth} + \\addtolength{\\mynoticelength}{-2\\fboxsep} + \\addtolength{\\mynoticelength}{-2\\fboxrule} + \\begin{lrbox}{\\@tempboxa}\\begin{minipage}{\\mynoticelength}}{\\end{minipage}\\end{lrbox}% \\ifthenelse% {\\equal{\\py@noticetype}{note}}% {\\colorbox{NoteColor}{\\usebox{\\@tempboxa}}}% From ca7bfe2c8d9f3aee469a3a36110a95ebb511ee20 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 22 Aug 2016 11:04:49 -0300 Subject: [PATCH 51/51] docs-rst: add package adjustbox We need adjustbox to allow adjusting the size of tables that are bigger than the line width. There are quite a few of them at the media books. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet --- Documentation/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/conf.py b/Documentation/conf.py index 85e9f16b1d98..ab484e56e23c 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -326,6 +326,9 @@ latex_elements = { \\setromanfont{DejaVu Sans} \\setmonofont{DejaVu Sans Mono} + % To allow adjusting table sizes + \\usepackage{adjustbox} + ''' }