From bf56cc04ef97c8ec536e3fcd16fc57902cba339f Mon Sep 17 00:00:00 2001 From: Vaishali Thakkar Date: Tue, 24 May 2016 09:49:17 +0530 Subject: [PATCH 01/17] Coccinelle: noderef: Add new rules and correct the old rule Add new rules to detect the cases where sizeof is used in function calls as a argument. Also, for the patch mode third rule should behave same as second rule with arguments reversed. So, change that as well. Signed-off-by: Vaishali Thakkar Acked-by: Julia Lawall Signed-off-by: Michal Marek --- scripts/coccinelle/misc/noderef.cocci | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/coccinelle/misc/noderef.cocci b/scripts/coccinelle/misc/noderef.cocci index 80a831c91161..007f0de0c715 100644 --- a/scripts/coccinelle/misc/noderef.cocci +++ b/scripts/coccinelle/misc/noderef.cocci @@ -16,6 +16,7 @@ virtual patch @depends on patch@ expression *x; expression f; +expression i; type T; @@ @@ -30,15 +31,26 @@ f(...,(T)(x),...,sizeof( + *x ),...) | -f(...,sizeof(x),...,(T)( +f(...,sizeof( +- x ++ *x + ),...,(T)(x),...) +| +f(...,(T)(x),...,i*sizeof( - x + *x ),...) +| +f(...,i*sizeof( +- x ++ *x + ),...,(T)(x),...) ) @r depends on !patch@ expression *x; expression f; +expression i; position p; type T; @@ @@ -49,6 +61,10 @@ type T; *f(...,(T)(x),...,sizeof@p(x),...) | *f(...,sizeof@p(x),...,(T)(x),...) +| +*f(...,(T)(x),...,i*sizeof@p(x),...) +| +*f(...,i*sizeof@p(x),...,(T)(x),...) ) @script:python depends on org@ From 7a2358b3818691521c7df531415d1ea4d0398520 Mon Sep 17 00:00:00 2001 From: Deepa Dinamani Date: Sun, 12 Jun 2016 12:04:39 -0700 Subject: [PATCH 02/17] coccicheck: Allow for overriding spatch flags Documentation/coccinelle.txt suggests using the SPFLAGS make variable to pass additional options to spatch. Reorder the way SPFLAGS is added to FLAGS, to allow for options in the SPFLAGS to override the default --very-quiet option. Similarly, rearrage the FLAGS for org or report mode. This allows for overriding of the default --no-show-diff option through SPFLAGS. Signed-off-by: Deepa Dinamani Cc: Gilles Muller Acked-by: Nicolas Palix Acked-by: Julia Lawall Signed-off-by: Michal Marek --- scripts/coccicheck | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/coccicheck b/scripts/coccicheck index dd85a455b2ba..f6627863fdc3 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -25,7 +25,7 @@ else NPROC="$J" fi -FLAGS="$SPFLAGS --very-quiet" +FLAGS="--very-quiet $SPFLAGS" # spatch only allows include directories with the syntax "-I include" # while gcc also allows "-Iinclude" and "-include include" @@ -72,7 +72,7 @@ if [ "$MODE" = "chain" ] ; then echo 'All available modes will be tried (in that order): patch, report, context, org' fi elif [ "$MODE" = "report" -o "$MODE" = "org" ] ; then - FLAGS="$FLAGS --no-show-diff" + FLAGS="--no-show-diff $FLAGS" fi if [ "$ONLINE" = "0" ] ; then From 6dd9379e8f327e70d182b15be3ba21aa2b5d2cba Mon Sep 17 00:00:00 2001 From: Yann Droneaud Date: Mon, 23 May 2016 17:07:19 +0200 Subject: [PATCH 03/17] coccinelle: also catch kzfree() issues Since commit 3ef0e5ba4673 ('slab: introduce kzfree()'), kfree() is no more the only function to be considered: kzfree() should be recognized too. In particular, kzfree() must not be called on memory allocated through devm_*() functions. Cc: Johannes Weiner Acked-by: Julia Lawall Signed-off-by: Yann Droneaud Signed-off-by: Michal Marek --- scripts/coccinelle/free/devm_free.cocci | 2 ++ scripts/coccinelle/free/ifnullfree.cocci | 4 +++- scripts/coccinelle/free/kfree.cocci | 18 +++++++++++++++--- scripts/coccinelle/free/kfreeaddr.cocci | 6 +++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 3d9349012bb3..83c03adec1c5 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -48,6 +48,8 @@ position p; ( * kfree@p(x) | +* kzfree@p(x) +| * free_irq@p(x) | * iounmap@p(x) diff --git a/scripts/coccinelle/free/ifnullfree.cocci b/scripts/coccinelle/free/ifnullfree.cocci index 52bd235286fa..14a4cd98e83b 100644 --- a/scripts/coccinelle/free/ifnullfree.cocci +++ b/scripts/coccinelle/free/ifnullfree.cocci @@ -19,6 +19,8 @@ expression E; - if (E != NULL) ( kfree(E); +| + kzfree(E); | debugfs_remove(E); | @@ -39,7 +41,7 @@ position p; @@ * if (E != NULL) -* \(kfree@p\|debugfs_remove@p\|debugfs_remove_recursive@p\| +* \(kfree@p\|kzfree@p\|debugfs_remove@p\|debugfs_remove_recursive@p\| * usb_free_urb@p\|kmem_cache_destroy@p\|mempool_destroy@p\| * dma_pool_destroy@p\)(E); diff --git a/scripts/coccinelle/free/kfree.cocci b/scripts/coccinelle/free/kfree.cocci index 577b78056990..ac438da4fd7b 100644 --- a/scripts/coccinelle/free/kfree.cocci +++ b/scripts/coccinelle/free/kfree.cocci @@ -20,7 +20,11 @@ expression E; position p1; @@ -kfree@p1(E) +( +* kfree@p1(E) +| +* kzfree@p1(E) +) @print expression@ constant char [] c; @@ -60,7 +64,11 @@ position ok; @@ while (1) { ... - kfree@ok(E) +( +* kfree@ok(E) +| +* kzfree@ok(E) +) ... when != break; when != goto l; when forall @@ -74,7 +82,11 @@ statement S; position free.p1!=loop.ok,p2!={print.p,sz.p}; @@ -kfree@p1(E,...) +( +* kfree@p1(E,...) +| +* kzfree@p1(E,...) +) ... ( iter(...,subE,...) S // no use diff --git a/scripts/coccinelle/free/kfreeaddr.cocci b/scripts/coccinelle/free/kfreeaddr.cocci index ce8aacc314cb..d46063b1db8b 100644 --- a/scripts/coccinelle/free/kfreeaddr.cocci +++ b/scripts/coccinelle/free/kfreeaddr.cocci @@ -16,7 +16,11 @@ identifier f; position p; @@ +( * kfree@p(&e->f) +| +* kzfree@p(&e->f) +) @script:python depends on org@ p << r.p; @@ -28,5 +32,5 @@ cocci.print_main("kfree",p) p << r.p; @@ -msg = "ERROR: kfree of structure field" +msg = "ERROR: invalid free of structure field" coccilib.report.print_report(p[0],msg) From a720c0644d14011d3a1bcc8d1b36e80571ad2ce1 Mon Sep 17 00:00:00 2001 From: Yann Droneaud Date: Mon, 23 May 2016 17:07:20 +0200 Subject: [PATCH 04/17] coccinelle: recognize more devm_* memory allocation functions Updates free/devm_free.cocci to recognize functions added by: - commit 64c862a839a8 ('devres: add kernel standard devm_k.alloc functions') - commit e31108cad3de ('devres: introduce API "devm_kstrdup"') - commit 3046365bb470 ('devres: introduce API "devm_kmemdup') - commit 43339bed7010 ('devres: Add devm_get_free_pages API') - commit 75f2a4ead5d5 ('devres: Add devm_kasprintf and devm_kvasprintf API') See also Documentation/driver-model/devres.txt Cc: Joe Perches Cc: Manish Badarkhe Cc: Srinivas Pandruvada Cc: Eli Billauer Cc: Himangi Saraogi Cc: Geert Uytterhoeven Cc: Wolfram Sang Cc: Daniel Thompson Acked-by: Julia Lawall Signed-off-by: Yann Droneaud Signed-off-by: Michal Marek --- scripts/coccinelle/free/devm_free.cocci | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 83c03adec1c5..3794cd97494b 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -29,7 +29,23 @@ expression x; @@ ( + x = devm_kmalloc(...) +| + x = devm_kvasprintf(...) +| + x = devm_kasprintf(...) +| x = devm_kzalloc(...) +| + x = devm_kmalloc_array(...) +| + x = devm_kcalloc(...) +| + x = devm_kstrdup(...) +| + x = devm_kmemdup(...) +| + x = devm_get_free_pages(...) | x = devm_request_irq(...) | @@ -50,6 +66,10 @@ position p; | * kzfree@p(x) | +* free_pages@p(x, ...) +| +* free_page@p(x) +| * free_irq@p(x) | * iounmap@p(x) From b7b2ee41f300b69c67c798df0cd5b8648bcb26a3 Mon Sep 17 00:00:00 2001 From: Yann Droneaud Date: Mon, 23 May 2016 17:07:21 +0200 Subject: [PATCH 05/17] coccinelle: catch krealloc() on devm_*() allocated memory krealloc() must not be used against devm_*() allocated memory regions: - if a bigger memory is to be allocated, krealloc() and __krealloc() could return a different pointer than the one given to them, creating a memory region which is not managed, thus it will not be automatically released on device removal. - if a bigger memory is to be allocated, krealloc() could kfree() the managed memory region which is passed to it. The old pointer is left registered as a resource for the device. On device removal, this dangling pointer will be used and an unrelated memory region could be released. - if the requested size is equal to 0, krealloc() can also just behave like kfree(). Here too, the old pointer is kept associated with the device. On device removal, this invalid pointer will be used and an unrelated memory region could be released. For all these reasons, krealloc() must not be used on a pointer returned by devm_*() functions. Cc: Tejun Heo Cc: Pekka Enberg Acked-by: Julia Lawall Signed-off-by: Yann Droneaud Signed-off-by: Michal Marek --- scripts/coccinelle/free/devm_free.cocci | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/coccinelle/free/devm_free.cocci b/scripts/coccinelle/free/devm_free.cocci index 3794cd97494b..c990d2c7ee16 100644 --- a/scripts/coccinelle/free/devm_free.cocci +++ b/scripts/coccinelle/free/devm_free.cocci @@ -66,6 +66,10 @@ position p; | * kzfree@p(x) | +* __krealloc@p(x, ...) +| +* krealloc@p(x, ...) +| * free_pages@p(x, ...) | * free_page@p(x) From 15f6d337159b2a9fdad8c0bef50ec826593ed5d2 Mon Sep 17 00:00:00 2001 From: Wilfried Klaebe Date: Tue, 28 Jun 2016 12:21:33 +0000 Subject: [PATCH 06/17] builddeb: really include objtool binary in headers package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On May 4th, Bjørn Mork provided patch 697bbc7b8320 ("builddeb: include objtool binary in headers package"). However, that one only works if $srctree=$objtree, because the objtool binaries are not written to the srctree, but to the objtree. Signed-off-by: Wilfried Klaebe Fixes: 697bbc7b8320 ("builddeb: include objtool binary in headers package") Signed-off-by: Michal Marek --- scripts/package/builddeb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 86e56fef7473..202d6e7db859 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -322,12 +322,12 @@ fi # Build kernel header package (cd $srctree; find . -name Makefile\* -o -name Kconfig\* -o -name \*.pl) > "$objtree/debian/hdrsrcfiles" -if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then - (cd $srctree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrsrcfiles" -fi (cd $srctree; find arch/*/include include scripts -type f) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find arch/$SRCARCH -name module.lds -o -name Kbuild.platforms -o -name Platform) >> "$objtree/debian/hdrsrcfiles" (cd $srctree; find $(find arch/$SRCARCH -name include -o -name scripts -type d) -type f) >> "$objtree/debian/hdrsrcfiles" +if grep -q '^CONFIG_STACK_VALIDATION=y' $KCONFIG_CONFIG ; then + (cd $objtree; find tools/objtool -type f -executable) >> "$objtree/debian/hdrobjfiles" +fi (cd $objtree; find arch/$SRCARCH/include Module.symvers include scripts -type f) >> "$objtree/debian/hdrobjfiles" destdir=$kernel_headers_dir/usr/src/linux-headers-$version mkdir -p "$destdir" From 13d948653372987388a09c259758abd44ed6e7f2 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:51 -0700 Subject: [PATCH 07/17] coccicheck: move spatch binary check up This has no functional changes. This is being done to enable us to later use spatch binary for some flag checking for certain features early on. Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- scripts/coccicheck | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/coccicheck b/scripts/coccicheck index f6627863fdc3..f137b04dfdd3 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -7,6 +7,11 @@ SPATCH="`which ${SPATCH:=spatch}`" +if [ ! -x "$SPATCH" ]; then + echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' + exit 1 +fi + trap kill_running SIGTERM SIGINT declare -a SPATCH_PID @@ -51,11 +56,6 @@ if [ "$KBUILD_EXTMOD" != "" ] ; then OPTIONS="--patch $srctree $OPTIONS" fi -if [ ! -x "$SPATCH" ]; then - echo 'spatch is part of the Coccinelle project and is available at http://coccinelle.lip6.fr/' - exit 1 -fi - if [ "$MODE" = "" ] ; then if [ "$ONLINE" = "0" ] ; then echo 'You have not explicitly specified the mode to use. Using default "report" mode.' From 8e826ad52b751ca34d5153533f4f58ec38cf4799 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:52 -0700 Subject: [PATCH 08/17] coccicheck: make SPFLAGS more useful SPFLAGS is set early, it means that any heuristics done on coccicheck cannot be overridden currently. Move SPFLAGS after OPTIONS and set this at the end. This lets you override any heuristics as coccinelle treats conflicts by only listening to the last option that makes sense. v3: this patch was added in the v3 series v4: Update Documentation/coccinelle.txt explaining how SPFLAGS works as well. Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 3 ++- scripts/coccicheck | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 7f773d51fdd9..bb9632c20cfb 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -146,7 +146,8 @@ MODE variable explained above. ~~~~~~~~~~~~~~~~~~ Additional flags can be passed to spatch through the SPFLAGS -variable. +variable. This works as Coccinelle respects the last flags +given to it when options are in conflict. make SPFLAGS=--use-glimpse coccicheck make SPFLAGS=--use-idutils coccicheck diff --git a/scripts/coccicheck b/scripts/coccicheck index f137b04dfdd3..5319fae910b4 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -30,7 +30,7 @@ else NPROC="$J" fi -FLAGS="--very-quiet $SPFLAGS" +FLAGS="--very-quiet" # spatch only allows include directories with the syntax "-I include" # while gcc also allows "-Iinclude" and "-include include" @@ -106,6 +106,9 @@ kill_running() { done } +# You can override heuristics with SPFLAGS, these must always go last +OPTIONS="$OPTIONS $SPFLAGS" + coccinelle () { COCCI="$1" From c930a1b23bb7a22439cf505db130a8db60e22688 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:53 -0700 Subject: [PATCH 09/17] coccicheck: enable parmap support Coccinelle has had parmap support since 1.0.2, this means it supports --jobs, enabling built-in multithreaded functionality, instead of needing one to script it out. Just look for --jobs in the help output to determine if this is supported and use it only if your number of processors detected is > 1. If parmap is enabled also enable the load balancing to be dynamic, so that if a thread finishes early we keep feeding it. stderr is currently sent to /dev/null, addressing a way to capture that will be addressed next. If --jobs is not supported we fallback to the old mechanism. We expect to deprecate the old mechanism as soon as we can get confirmation all users are ready. While at it propagate back into the shell script any coccinelle error code. When used in serialized mode where all cocci files are run this also stops processing if an error has occured. This lets us handle some errors in coccinelle cocci files and if they bail out we should inspect the errors. This will be more useful later to help annotate coccinelle version dependency requirements. This will let you run only SmPL files that your system supports. Extend Documentation/coccinelle.txt as well. As a small example, prior to this change, on an 8-core system: Before: $ export COCCI=scripts/coccinelle/free/kfree.cocci $ time make coccicheck MODE=report ... real 29m14.912s user 103m1.796s sys 0m4.464s After: real 16m22.435s user 128m30.060s sys 0m2.712s v4: o expand Documentation/coccinelle.txt to reflect parmap support info o update commit log to reflect what we actually do now with stderr o split out DEBUG_FILE use into another patch o detect number of CPUs and if its 1 then skip parmap support, note that if you still support parmap, but have 1 CPU you will also go through the new branches, so the old complex multithreaded process is skipped as well. v3: o move USE_JOBS to avoid being overriden v2: o redirect coccinelle stderr to /dev/null by default and only if DEBUG_FILE is used do we pass it to a file o fix typo of paramap/parmap Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 15 +++++++++++++++ scripts/coccicheck | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index bb9632c20cfb..64daaf18874b 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -94,11 +94,26 @@ To enable verbose messages set the V= variable, for example: make coccicheck MODE=report V=1 + Coccinelle parallelization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + By default, coccicheck tries to run as parallel as possible. To change the parallelism, set the J= variable. For example, to run across 4 CPUs: make coccicheck MODE=report J=4 +As of Coccinelle 1.0.2 Coccinelle uses Ocaml parmap for parallelization, +if support for this is detected you will benefit from parmap parallelization. + +When parmap is enabled coccicheck will enable dynamic load balancing by using +'--chunksize 1' argument, this ensures we keep feeding threads with work +one by one, so that we avoid the situation where most work gets done by only +a few threads. With dynamic load balancing, if a thread finishes early we keep +feeding it more work. + +When parmap is enabled, if an error occurs in Coccinelle, this error +value is propagated back, the return value of the 'make coccicheck' +captures this return value. Using Coccinelle with a single semantic patch ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/coccicheck b/scripts/coccicheck index 5319fae910b4..4b65a0fd50a1 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -12,8 +12,8 @@ if [ ! -x "$SPATCH" ]; then exit 1 fi -trap kill_running SIGTERM SIGINT -declare -a SPATCH_PID +USE_JOBS="no" +$SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes" # The verbosity may be set by the environmental parameter V= # as for example with 'make V=1 coccicheck' @@ -56,6 +56,16 @@ if [ "$KBUILD_EXTMOD" != "" ] ; then OPTIONS="--patch $srctree $OPTIONS" fi +# You can override by using SPFLAGS +if [ "$USE_JOBS" = "no" ]; then + trap kill_running SIGTERM SIGINT + declare -a SPATCH_PID +elif [ "$NPROC" != "1" ]; then + # Using 0 should work as well, refer to _SC_NPROCESSORS_ONLN use on + # https://github.com/rdicosmo/parmap/blob/master/setcore_stubs.c + OPTIONS="$OPTIONS --jobs $NPROC --chunksize 1" +fi + if [ "$MODE" = "" ] ; then if [ "$ONLINE" = "0" ] ; then echo 'You have not explicitly specified the mode to use. Using default "report" mode.' @@ -82,7 +92,18 @@ if [ "$ONLINE" = "0" ] ; then echo '' fi -run_cmd() { +run_cmd_parmap() { + if [ $VERBOSE -ne 0 ] ; then + echo "Running ($NPROC in parallel): $@" + fi + $@ 2>/dev/null + if [[ $? -ne 0 ]]; then + echo "coccicheck failed" + exit $? + fi +} + +run_cmd_old() { local i if [ $VERBOSE -ne 0 ] ; then echo "Running ($NPROC in parallel): $@" @@ -97,6 +118,14 @@ run_cmd() { wait } +run_cmd() { + if [ "$USE_JOBS" = "yes" ]; then + run_cmd_parmap $@ + else + run_cmd_old $@ + fi +} + kill_running() { for i in $(seq 0 $(( NPROC - 1 )) ); do if [ $VERBOSE -eq 2 ] ; then From be1fa90066644c5a7fdf4a08767c4a359f95096f Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:54 -0700 Subject: [PATCH 10/17] coccicheck: add support for DEBUG_FILE Enable to capture stderr via a DEBUG_FILE variable passed to coccicheck. You can now do: $ rm -f cocci.err $ export COCCI=scripts/coccinelle/free/kfree.cocci $ make coccicheck MODE=report DEBUG_FILE=cocci.err ... $ cat cocci.err This will be come more useful once we add support to use more things which would go into stderr, such as profiling. That will be done separately in another commit. Expand Documentation/coccinelle.txt with details. Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 20 ++++++++++++++++++++ scripts/coccicheck | 10 +++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 64daaf18874b..2516c5ef1691 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -157,6 +157,26 @@ semantic patch as shown in the previous section. The "report" mode is the default. You can select another one with the MODE variable explained above. + Debugging Coccinelle SmPL patches +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Using coccicheck is best as it provides in the spatch command line +include options matching the options used when we compile the kernel. +You can learn what these options are by using V=1, you could then +manually run Coccinelle with debug options added. + +Alternatively you can debug running Coccinelle against SmPL patches +by asking for stderr to be redirected to stderr, by default stderr +is redirected to /dev/null, if you'd like to capture stderr you +can specify the DEBUG_FILE="file.txt" option to coccicheck. For +instance: + + rm -f cocci.err + make coccicheck COCCI=scripts/coccinelle/free/kfree.cocci MODE=report DEBUG_FILE=cocci.err + cat cocci.err + +DEBUG_FILE support is only supported when using coccinelle >= 1.2. + Additional flags ~~~~~~~~~~~~~~~~~~ diff --git a/scripts/coccicheck b/scripts/coccicheck index 4b65a0fd50a1..3f0bb3f0fddc 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -96,7 +96,15 @@ run_cmd_parmap() { if [ $VERBOSE -ne 0 ] ; then echo "Running ($NPROC in parallel): $@" fi - $@ 2>/dev/null + if [ "$DEBUG_FILE" != "/dev/null" -a "$DEBUG_FILE" != "" ]; then + if [ -f $DEBUG_FILE ]; then + echo "Debug file $DEBUG_FILE exists, bailing" + exit + fi + else + DEBUG_FILE="/dev/null" + fi + $@ 2>$DEBUG_FILE if [[ $? -ne 0 ]]; then echo "coccicheck failed" exit $? From 5c384dba979f7aa57d1f09a6ca1a7cd486e7caba Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:55 -0700 Subject: [PATCH 11/17] coccicheck: replace --very-quiet with --quiet when debugging When debugging (using --profile or --show-trying) you want to avoid supressing output, use --quiet instead. While at it, extend documentation for SPFLAGS use. For instance one can use: $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci $ make coccicheck DEBUG_FILE="poo.err" MODE=report SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c Expand Documentation/coccinelle.txt as well. v4: expand Documentation/coccinelle.txt v3: rebased, resolve conflicts, expand Documentation/coccinelle.txt v2: use egrep instead of the *"=--option"* check, this doesn't work for disjunctions. Signed-off-by: Luis R. Rodriguez Acked-by: Julia Lawall Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 12 ++++++++++++ scripts/coccicheck | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 2516c5ef1691..66e822f8caee 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -175,6 +175,18 @@ instance: make coccicheck COCCI=scripts/coccinelle/free/kfree.cocci MODE=report DEBUG_FILE=cocci.err cat cocci.err +You can use SPFLAGS to add debugging flags, for instance you may want to +add both --profile --show-trying to SPFLAGS when debugging. For instance +you may want to use: + + rm -f err.log + export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci + make coccicheck DEBUG_FILE="err.log" MODE=report SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c + +err.log will now have the profiling information, while stdout will +provide some progress information as Coccinelle moves forward with +work. + DEBUG_FILE support is only supported when using coccinelle >= 1.2. Additional flags diff --git a/scripts/coccicheck b/scripts/coccicheck index 3f0bb3f0fddc..081ba5bff79c 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -32,6 +32,27 @@ fi FLAGS="--very-quiet" +# You can use SPFLAGS to append extra arguments to coccicheck or override any +# heuristics done in this file as Coccinelle accepts the last options when +# options conflict. +# +# A good example for use of SPFLAGS is if you want to debug your cocci script, +# you can for instance use the following: +# +# $ export COCCI=scripts/coccinelle/misc/irqf_oneshot.cocci +# $ make coccicheck MODE=report DEBUG_FILE="all.err" SPFLAGS="--profile --show-trying" M=./drivers/mfd/arizona-irq.c +# +# "--show-trying" should show you what rule is being processed as it goes to +# stdout, you do not need a debug file for that. The profile output will be +# be sent to stdout, if you provide a DEBUG_FILE the profiling data can be +# inspected there. +# +# --profile will not output if --very-quiet is used, so avoid it. +echo $SPFLAGS | egrep -e "--profile|--show-trying" 2>&1 > /dev/null +if [ $? -eq 0 ]; then + FLAGS="--quiet" +fi + # spatch only allows include directories with the syntax "-I include" # while gcc also allows "-Iinclude" and "-include include" COCCIINCLUDE=${LINUXINCLUDE//-I/-I } From dd951fc1b67b6511d6c8f0449f3d451df374ebce Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:56 -0700 Subject: [PATCH 12/17] scripts: add Linux .cocciconfig for coccinelle Coccinelle supports reading .cocciconfig, the order of precedence for variables for .cocciconfig is as follows: o Your current user's home directory is processed first o Your directory from which spatch is called is processed next o The directory provided with the --dir option is processed last, if used Since coccicheck runs through make, it naturally runs from the kernel proper dir, as such the second rule above would be implied for picking up a .cocciconfig when using 'make coccicheck'. 'make coccicheck' also supports using M= targets.If you do not supply any M= target, it is assumed you want to target the entire kernel. The kernel coccicheck script has: if [ "$KBUILD_EXTMOD" = "" ] ; then OPTIONS="--dir $srctree $COCCIINCLUDE" else OPTIONS="--dir $KBUILD_EXTMOD $COCCIINCLUDE" fi KBUILD_EXTMOD is set when an explicit target with M= is used. For both cases the spatch --dir argument is used, as such third rule applies when whether M= is used or not, and when M= is used the target directory can have its own .cocciconfig file. When M= is not passed as an argument to coccicheck the target directory is the same as the directory from where spatch was called. If not using the kernel's coccicheck target, keep the above precedence order logic of .cocciconfig reading. If using the kernel's coccicheck target, override any of the kernel's .coccicheck's settings using SPFLAGS. We help Coccinelle when used against Linux with a set of sensible defaults options for Linux with our own Linux .cocciconfig. This hints to coccinelle git can be used for 'git grep' queries over coccigrep. A timeout of 200 seconds should suffice for now. The options picked up by coccinelle when reading a .cocciconfig do not appear as arguments to spatch processes running on your system, to confirm what options will be used by Coccinelle run: spatch --print-options-only You can override with your own preferred index option by using SPFLAGS. Coccinelle supports both glimpse and idutils. Glimpse had historically provided the best performance, however recent benchmarks reveal idutils is performing just as well. Due to some recent fixes however you however will need at least coccinelle >= 1.0.6 if using idutils. Coccinelle carries a script scripts/idutils_index.sh which creates the idutils database with as follows: mkid -i C --output .id-utils.index If using just "--use-idutils" coccinelle expects your idutils database to be on the top level of the kernel as a file named ".id-utils.index". If you do not use this you can symlink your database file to it, or you can specify the database file following the "--use-idutils" argument. Examples: make SPFLAGS=--use-idutils coccicheck This assumes you have $srctree/.id-utils.index, where $srctree is the top level of the kernel. make SPFLAGS="--use-idutils /full-path/to/ID" coccicheck Here you specify the full path of the idutils ID database. Using .cocciconfig is possible, however given the order of precedence followed by Coccinelle, and since the kernel now carries its own .cocciconfig, you will need to use SPFLAGS to use idutils if desired. v4: o Recommend upgrade for using idutils with coccinelle due to some recent fixes. o Refer to using --print-options-only for testing what options are picked up by .cocciconfig reading. o Expand commit log considerably explaining *why* .cocconfig from two precedence rules are used when using coccicheck, and how to properly override these if needed. o Expand Documentation/coccinelle.txt v3: Expand commit log a bit more Signed-off-by: Luis R. Rodriguez Acked-by: Julia Lawall Signed-off-by: Michal Marek --- .cocciconfig | 3 ++ .gitignore | 1 + Documentation/coccinelle.txt | 70 ++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 .cocciconfig diff --git a/.cocciconfig b/.cocciconfig new file mode 100644 index 000000000000..43967c6b2015 --- /dev/null +++ b/.cocciconfig @@ -0,0 +1,3 @@ +[spatch] + options = --timeout 200 + options = --use-gitgrep diff --git a/.gitignore b/.gitignore index 0c320bf02586..038ae7a397cd 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ Module.symvers # !.gitignore !.mailmap +!.cocciconfig # # Generated include files diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 66e822f8caee..b50ac7e126e8 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -189,6 +189,60 @@ work. DEBUG_FILE support is only supported when using coccinelle >= 1.2. + .cocciconfig support +~~~~~~~~~~~~~~~~~~~~~~ + +Coccinelle supports reading .cocciconfig for default Coccinelle options that +should be used every time spatch is spawned, the order of precedence for +variables for .cocciconfig is as follows: + + o Your current user's home directory is processed first + o Your directory from which spatch is called is processed next + o The directory provided with the --dir option is processed last, if used + +Since coccicheck runs through make, it naturally runs from the kernel +proper dir, as such the second rule above would be implied for picking up a +.cocciconfig when using 'make coccicheck'. + +'make coccicheck' also supports using M= targets.If you do not supply +any M= target, it is assumed you want to target the entire kernel. +The kernel coccicheck script has: + + if [ "$KBUILD_EXTMOD" = "" ] ; then + OPTIONS="--dir $srctree $COCCIINCLUDE" + else + OPTIONS="--dir $KBUILD_EXTMOD $COCCIINCLUDE" + fi + +KBUILD_EXTMOD is set when an explicit target with M= is used. For both cases +the spatch --dir argument is used, as such third rule applies when whether M= +is used or not, and when M= is used the target directory can have its own +.cocciconfig file. When M= is not passed as an argument to coccicheck the +target directory is the same as the directory from where spatch was called. + +If not using the kernel's coccicheck target, keep the above precedence +order logic of .cocciconfig reading. If using the kernel's coccicheck target, +override any of the kernel's .coccicheck's settings using SPFLAGS. + +We help Coccinelle when used against Linux with a set of sensible defaults +options for Linux with our own Linux .cocciconfig. This hints to coccinelle +git can be used for 'git grep' queries over coccigrep. A timeout of 200 +seconds should suffice for now. + +The options picked up by coccinelle when reading a .cocciconfig do not appear +as arguments to spatch processes running on your system, to confirm what +options will be used by Coccinelle run: + + spatch --print-options-only + +You can override with your own preferred index option by using SPFLAGS. Take +note that when there are conflicting options Coccinelle takes precedence for +the last options passed. Using .cocciconfig is possible to use idutils, however +given the order of precedence followed by Coccinelle, since the kernel now +carries its own .cocciconfig, you will need to use SPFLAGS to use idutils if +desired. See below section "Additional flags" for more details on how to use +idutils. + Additional flags ~~~~~~~~~~~~~~~~~~ @@ -197,8 +251,24 @@ variable. This works as Coccinelle respects the last flags given to it when options are in conflict. make SPFLAGS=--use-glimpse coccicheck + +Coccinelle supports idutils as well but requires coccinelle >= 1.0.6. +When no ID file is specified coccinelle assumes your ID database file +is in the file .id-utils.index on the top level of the kernel, coccinelle +carries a script scripts/idutils_index.sh which creates the database with + + mkid -i C --output .id-utils.index + +If you have another database filename you can also just symlink with this +name. + make SPFLAGS=--use-idutils coccicheck +Alternatively you can specify the database filename explicitly, for +instance: + + make SPFLAGS="--use-idutils /full-path/to/ID" coccicheck + See spatch --help to learn more about spatch options. Note that the '--use-glimpse' and '--use-idutils' options From a9e064c00413fb59e7236be81578024ab0ebe6b2 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:57 -0700 Subject: [PATCH 13/17] coccicheck: add support for requring a coccinelle version Enable Coccinelle SmPL patches to require a specific version of Coccinelle. In the event that the version does not match we just inform the user, if the user asked to go through all SmPL patches we just inform them of the need for a new version of coccinelle for the SmPL patch and continue on with the rest. This uses the simple kernel scripts/ld-version.sh to create a weight on the version provided by spatch. The -dirty attribute is ignored if supplied, the benefit of scripts/ld-version.sh is it has a long history and well tested. While at it, document the // Options stuff as well. v4: Document // Options and // Requires as well on Documentation/coccinelle.txt. Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 19 +++++++++++++++++++ scripts/coccicheck | 14 ++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index b50ac7e126e8..1c26908ebc16 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -277,6 +277,25 @@ thus active by default. However, by indexing the code with one of these tools, and according to the cocci file used, spatch could proceed the entire code base more quickly. + SmPL patch specific options +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SmPL patches can have their own requirements for options passed +to Coccinelle. SmPL patch specific options can be provided by +providing them at the top of the SmPL patch, for instance: + +// Options: --no-includes --include-headers + + SmPL patch Coccinelle requirements +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +As Coccinelle features get added some more advanced SmPL patches +may require newer versions of Coccinelle. If an SmPL patch requires +at least a version of Coccinelle, this can be specified as follows, +as an example if requiring at least Coccinelle >= 1.0.5: + +// Requires: 1.0.5 + Proposing new semantic patches ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/coccicheck b/scripts/coccicheck index 081ba5bff79c..f9293ab04a8b 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -5,6 +5,7 @@ # version 1.0.0-rc11. # +DIR="$(dirname $(readlink -f $0))/.." SPATCH="`which ${SPATCH:=spatch}`" if [ ! -x "$SPATCH" ]; then @@ -12,6 +13,9 @@ if [ ! -x "$SPATCH" ]; then exit 1 fi +SPATCH_VERSION=$($SPATCH --version | head -1 | awk '{print $3}') +SPATCH_VERSION_NUM=$(echo $SPATCH_VERSION | ${DIR}/scripts/ld-version.sh) + USE_JOBS="no" $SPATCH --help | grep "\-\-jobs" > /dev/null && USE_JOBS="yes" @@ -171,6 +175,16 @@ coccinelle () { COCCI="$1" OPT=`grep "Option" $COCCI | cut -d':' -f2` + REQ=`grep "Requires" $COCCI | cut -d':' -f2 | sed "s| ||"` + REQ_NUM=$(echo $REQ | ${DIR}/scripts/ld-version.sh) + if [ "$REQ_NUM" != "0" ] ; then + if [ "$SPATCH_VERSION_NUM" -lt "$REQ_NUM" ] ; then + echo "Skipping coccinele SmPL patch: $COCCI" + echo "You have coccinelle: $SPATCH_VERSION" + echo "This SmPL patch requires: $REQ" + return + fi + fi # The option '--parse-cocci' can be used to syntactically check the SmPL files. # From c100d537b9a088caddb4e355847986b439b017c2 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:58 -0700 Subject: [PATCH 14/17] coccicheck: refer to Documentation/coccinelle.txt and wiki Refer to the Documentation/coccinelle.txt and supplemental documentation on the wiki: https://bottest.wiki.kernel.org/coccicheck This page shall always refer to the linux-next iteration of scripts/coccicheck. v4: only refer to the wiki as supplemental documentation, and also update Documentation/coccinelle.txt. Signed-off-by: Luis R. Rodriguez Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- Documentation/coccinelle.txt | 9 +++++++++ scripts/coccicheck | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Documentation/coccinelle.txt b/Documentation/coccinelle.txt index 1c26908ebc16..01fb1dae3163 100644 --- a/Documentation/coccinelle.txt +++ b/Documentation/coccinelle.txt @@ -38,6 +38,15 @@ as a regular user, and install it with sudo make install + Supplemental documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For supplemental documentation refer to the wiki: + +https://bottest.wiki.kernel.org/coccicheck + +The wiki documentation always refers to the linux-next version of the script. + Using Coccinelle on the Linux kernel ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/coccicheck b/scripts/coccicheck index f9293ab04a8b..c92c1528a54d 100755 --- a/scripts/coccicheck +++ b/scripts/coccicheck @@ -1,9 +1,10 @@ #!/bin/bash - +# Linux kernel coccicheck +# +# Read Documentation/coccinelle.txt # # This script requires at least spatch # version 1.0.0-rc11. -# DIR="$(dirname $(readlink -f $0))/.." SPATCH="`which ${SPATCH:=spatch}`" From cc65e823363834a7ff0a2eca9bef2bbef64135f7 Mon Sep 17 00:00:00 2001 From: "Luis R. Rodriguez" Date: Wed, 29 Jun 2016 15:14:59 -0700 Subject: [PATCH 15/17] scripts/coccinelle: require coccinelle >= 1.0.4 on device_node_continue.cocci Make use of the new Requires: tag to be able to specify coccinelle binary version requirements. The cocci file device_node_continue.cocci requires at least coccinelle 1.0.4. Signed-off-by: Luis R. Rodriguez Acked-by: Julia Lawall Acked-by: Nicolas Palix Signed-off-by: Michal Marek --- scripts/coccinelle/iterators/device_node_continue.cocci | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/coccinelle/iterators/device_node_continue.cocci b/scripts/coccinelle/iterators/device_node_continue.cocci index 38ab744a4037..a36c16db171b 100644 --- a/scripts/coccinelle/iterators/device_node_continue.cocci +++ b/scripts/coccinelle/iterators/device_node_continue.cocci @@ -5,8 +5,11 @@ // Copyright: (C) 2015 Julia Lawall, Inria. GPLv2. // URL: http://coccinelle.lip6.fr/ // Options: --no-includes --include-headers +// Requires: 1.0.4 // Keywords: for_each_child_of_node, etc. +// This uses a conjunction, which requires at least coccinelle >= 1.0.4 + virtual patch virtual context virtual org From ca617dc68b23407c4b7427d4f2fd71e0ba7ba81d Mon Sep 17 00:00:00 2001 From: Henning Schild Date: Fri, 22 Jul 2016 14:46:52 +0200 Subject: [PATCH 16/17] builddeb: fix file permissions before packaging Builddep is not very explicit about file permissions. Actually the file permissions in the package are largely influenced by the umask of the user cloning the git and building the package. If that umask does not set go+r the resulting linux-headers package will prevent non-root users from building out-of-tree modules. And that is probably just one unexpected effect. Being a packaging/install tool builddep should make sure the file permissions are set correctly and not just derived from a value that is never checked. This patch sets ugo read permissions for all packaged files and derives the executable bit for directories and executables from the file-owner. Signed-off-by: Henning Schild Signed-off-by: Michal Marek --- scripts/package/builddeb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 202d6e7db859..116ef00c0b82 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -26,6 +26,8 @@ create_package() { # Fix ownership and permissions chown -R root:root "$pdir" chmod -R go-w "$pdir" + # in case we are in a restrictive umask environment like 0077 + chmod -R a+rX "$pdir" # Create the package dpkg-gencontrol $forcearch -Vkernel:debarch="${debarch}" -p$pname -P"$pdir" From 4c586062b275dcddc18f521ac092cf0f600a36de Mon Sep 17 00:00:00 2001 From: Marcin Mielniczuk Date: Tue, 12 Jul 2016 21:42:35 +0200 Subject: [PATCH 17/17] Fix the Debian packaging script on systems with no codename When calling `make deb-pkg` on a system with no codename (for example Arch Linux), lsb_release sometimes outputs `n/a` as the codename. This breaks dpkg-parsechangelog, which can't process the changelog correctly. Signed-off-by: Marcin Mielniczuk Signed-off-by: Michal Marek --- scripts/package/builddeb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/package/builddeb b/scripts/package/builddeb index 116ef00c0b82..510add6d050c 100755 --- a/scripts/package/builddeb +++ b/scripts/package/builddeb @@ -240,7 +240,8 @@ maintainer="$name <$email>" # Try to determine distribution if [ -n "$KDEB_CHANGELOG_DIST" ]; then distribution=$KDEB_CHANGELOG_DIST -elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ]; then +# In some cases lsb_release returns the codename as n/a, which breaks dpkg-parsechangelog +elif distribution=$(lsb_release -cs 2>/dev/null) && [ -n "$distribution" ] && [ "$distribution" != "n/a" ]; then : # nothing to do in this case else distribution="unstable"