2005-10-30 16:03:12 -07:00
|
|
|
/*
|
2006-06-27 03:54:02 -06:00
|
|
|
* Read-Copy Update module-based torture test facility
|
2005-10-30 16:03:12 -07:00
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License as published by
|
|
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
|
|
* (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
*
|
2006-10-04 03:17:13 -06:00
|
|
|
* Copyright (C) IBM Corporation, 2005, 2006
|
2005-10-30 16:03:12 -07:00
|
|
|
*
|
|
|
|
* Authors: Paul E. McKenney <paulmck@us.ibm.com>
|
2009-09-18 11:28:19 -06:00
|
|
|
* Josh Triplett <josh@freedesktop.org>
|
2005-10-30 16:03:12 -07:00
|
|
|
*
|
|
|
|
* See also: Documentation/RCU/torture.txt
|
|
|
|
*/
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/kthread.h>
|
|
|
|
#include <linux/err.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/smp.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/sched.h>
|
2011-07-26 17:09:06 -06:00
|
|
|
#include <linux/atomic.h>
|
2005-10-30 16:03:12 -07:00
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/completion.h>
|
|
|
|
#include <linux/moduleparam.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/notifier.h>
|
2008-12-15 17:13:07 -07:00
|
|
|
#include <linux/reboot.h>
|
2007-07-17 05:03:35 -06:00
|
|
|
#include <linux/freezer.h>
|
2005-10-30 16:03:12 -07:00
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/delay.h>
|
|
|
|
#include <linux/stat.h>
|
2006-10-04 03:17:03 -06:00
|
|
|
#include <linux/srcu.h>
|
2008-04-29 01:59:25 -06:00
|
|
|
#include <linux/slab.h>
|
2008-10-20 11:23:38 -06:00
|
|
|
#include <asm/byteorder.h>
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
MODULE_LICENSE("GPL");
|
2006-10-04 03:17:13 -06:00
|
|
|
MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
|
2009-09-18 11:28:19 -06:00
|
|
|
"Josh Triplett <josh@freedesktop.org>");
|
2005-10-30 16:03:12 -07:00
|
|
|
|
2006-10-03 15:26:16 -06:00
|
|
|
static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */
|
2006-10-04 03:17:13 -06:00
|
|
|
static int nfakewriters = 4; /* # fake writer threads */
|
2006-01-08 02:03:42 -07:00
|
|
|
static int stat_interval; /* Interval between stats, in seconds. */
|
2005-10-30 16:03:12 -07:00
|
|
|
/* Defaults to "only at end of test". */
|
2006-01-08 02:03:42 -07:00
|
|
|
static int verbose; /* Print more debug info. */
|
|
|
|
static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */
|
2008-06-18 06:21:44 -06:00
|
|
|
static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
|
|
|
|
static int stutter = 5; /* Start/stop testing interval (in sec) */
|
2008-06-25 13:24:52 -06:00
|
|
|
static int irqreader = 1; /* RCU readers from irq (timers). */
|
2011-11-04 12:44:12 -06:00
|
|
|
static int fqs_duration; /* Duration of bursts (us), 0 to disable. */
|
|
|
|
static int fqs_holdoff; /* Hold time within burst (us). */
|
2010-01-04 16:09:10 -07:00
|
|
|
static int fqs_stutter = 3; /* Wait time between bursts (s). */
|
2011-11-16 18:48:21 -07:00
|
|
|
static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */
|
2011-11-04 12:44:12 -06:00
|
|
|
static int shutdown_secs; /* Shutdown time (s). <=0 for no shutdown. */
|
2010-09-02 17:16:14 -06:00
|
|
|
static int test_boost = 1; /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
|
|
|
|
static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
|
|
|
|
static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
|
2006-10-04 03:17:15 -06:00
|
|
|
static char *torture_type = "rcu"; /* What RCU implementation to torture. */
|
2005-10-30 16:03:12 -07:00
|
|
|
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(nreaders, int, 0444);
|
2005-10-30 16:03:12 -07:00
|
|
|
MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(nfakewriters, int, 0444);
|
2006-10-04 03:17:13 -06:00
|
|
|
MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
|
2011-07-21 17:00:17 -06:00
|
|
|
module_param(stat_interval, int, 0644);
|
2005-10-30 16:03:12 -07:00
|
|
|
MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(verbose, bool, 0444);
|
2005-10-30 16:03:12 -07:00
|
|
|
MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(test_no_idle_hz, bool, 0444);
|
2006-01-08 02:03:42 -07:00
|
|
|
MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(shuffle_interval, int, 0444);
|
2006-01-08 02:03:42 -07:00
|
|
|
MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
|
2008-06-18 06:21:44 -06:00
|
|
|
module_param(stutter, int, 0444);
|
|
|
|
MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
|
2008-06-25 13:24:52 -06:00
|
|
|
module_param(irqreader, int, 0444);
|
|
|
|
MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
|
2010-01-04 16:09:10 -07:00
|
|
|
module_param(fqs_duration, int, 0444);
|
|
|
|
MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
|
|
|
|
module_param(fqs_holdoff, int, 0444);
|
|
|
|
MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
|
|
|
|
module_param(fqs_stutter, int, 0444);
|
|
|
|
MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
|
2011-11-16 18:48:21 -07:00
|
|
|
module_param(onoff_interval, int, 0444);
|
|
|
|
MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
|
2011-11-04 12:44:12 -06:00
|
|
|
module_param(shutdown_secs, int, 0444);
|
|
|
|
MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
|
2010-09-02 17:16:14 -06:00
|
|
|
module_param(test_boost, int, 0444);
|
|
|
|
MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
|
|
|
|
module_param(test_boost_interval, int, 0444);
|
|
|
|
MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");
|
|
|
|
module_param(test_boost_duration, int, 0444);
|
|
|
|
MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");
|
2007-03-06 02:42:13 -07:00
|
|
|
module_param(torture_type, charp, 0444);
|
2006-10-04 03:17:03 -06:00
|
|
|
MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
|
2006-06-27 03:54:03 -06:00
|
|
|
|
|
|
|
#define TORTURE_FLAG "-torture:"
|
2005-10-30 16:03:12 -07:00
|
|
|
#define PRINTK_STRING(s) \
|
2006-06-27 03:54:03 -06:00
|
|
|
do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
|
2005-10-30 16:03:12 -07:00
|
|
|
#define VERBOSE_PRINTK_STRING(s) \
|
2006-06-27 03:54:03 -06:00
|
|
|
do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
|
2005-10-30 16:03:12 -07:00
|
|
|
#define VERBOSE_PRINTK_ERRSTRING(s) \
|
2006-06-27 03:54:03 -06:00
|
|
|
do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
static char printk_buf[4096];
|
|
|
|
|
|
|
|
static int nrealreaders;
|
|
|
|
static struct task_struct *writer_task;
|
2006-10-04 03:17:13 -06:00
|
|
|
static struct task_struct **fakewriter_tasks;
|
2005-10-30 16:03:12 -07:00
|
|
|
static struct task_struct **reader_tasks;
|
|
|
|
static struct task_struct *stats_task;
|
2006-01-08 02:03:42 -07:00
|
|
|
static struct task_struct *shuffler_task;
|
2008-06-18 06:21:44 -06:00
|
|
|
static struct task_struct *stutter_task;
|
2010-01-04 16:09:10 -07:00
|
|
|
static struct task_struct *fqs_task;
|
2010-09-02 17:16:14 -06:00
|
|
|
static struct task_struct *boost_tasks[NR_CPUS];
|
2011-11-04 12:44:12 -06:00
|
|
|
static struct task_struct *shutdown_task;
|
2011-11-16 18:48:21 -07:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
static struct task_struct *onoff_task;
|
|
|
|
#endif /* #ifdef CONFIG_HOTPLUG_CPU */
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
#define RCU_TORTURE_PIPE_LEN 10
|
|
|
|
|
|
|
|
struct rcu_torture {
|
|
|
|
struct rcu_head rtort_rcu;
|
|
|
|
int rtort_pipe_count;
|
|
|
|
struct list_head rtort_free;
|
2005-11-18 02:10:50 -07:00
|
|
|
int rtort_mbtest;
|
2005-10-30 16:03:12 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
static LIST_HEAD(rcu_torture_freelist);
|
2010-09-19 22:06:14 -06:00
|
|
|
static struct rcu_torture __rcu *rcu_torture_current;
|
2011-04-03 22:33:51 -06:00
|
|
|
static unsigned long rcu_torture_current_version;
|
2005-10-30 16:03:12 -07:00
|
|
|
static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
|
|
|
|
static DEFINE_SPINLOCK(rcu_torture_lock);
|
|
|
|
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
|
|
|
|
{ 0 };
|
|
|
|
static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
|
|
|
|
{ 0 };
|
|
|
|
static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
|
2006-10-04 03:17:03 -06:00
|
|
|
static atomic_t n_rcu_torture_alloc;
|
|
|
|
static atomic_t n_rcu_torture_alloc_fail;
|
|
|
|
static atomic_t n_rcu_torture_free;
|
|
|
|
static atomic_t n_rcu_torture_mberror;
|
|
|
|
static atomic_t n_rcu_torture_error;
|
2010-09-02 17:16:14 -06:00
|
|
|
static long n_rcu_torture_boost_ktrerror;
|
|
|
|
static long n_rcu_torture_boost_rterror;
|
|
|
|
static long n_rcu_torture_boost_failure;
|
|
|
|
static long n_rcu_torture_boosts;
|
2009-09-18 11:28:19 -06:00
|
|
|
static long n_rcu_torture_timers;
|
2011-11-16 18:48:21 -07:00
|
|
|
static long n_offline_attempts;
|
|
|
|
static long n_offline_successes;
|
|
|
|
static long n_online_attempts;
|
|
|
|
static long n_online_successes;
|
2006-10-04 03:17:14 -06:00
|
|
|
static struct list_head rcu_torture_removed;
|
2009-03-30 22:05:16 -06:00
|
|
|
static cpumask_var_t shuffle_tmp_mask;
|
2005-10-30 16:03:12 -07:00
|
|
|
|
2009-09-18 11:28:19 -06:00
|
|
|
static int stutter_pause_test;
|
2008-06-18 06:21:44 -06:00
|
|
|
|
2008-06-18 10:26:49 -06:00
|
|
|
#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
|
|
|
|
#define RCUTORTURE_RUNNABLE_INIT 1
|
|
|
|
#else
|
|
|
|
#define RCUTORTURE_RUNNABLE_INIT 0
|
|
|
|
#endif
|
|
|
|
int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
|
2011-11-04 14:24:07 -06:00
|
|
|
module_param(rcutorture_runnable, int, 0444);
|
|
|
|
MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot");
|
2008-06-18 10:26:49 -06:00
|
|
|
|
2011-04-18 00:45:23 -06:00
|
|
|
#if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
|
2010-09-02 17:16:14 -06:00
|
|
|
#define rcu_can_boost() 1
|
2011-04-18 00:45:23 -06:00
|
|
|
#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
|
2010-09-02 17:16:14 -06:00
|
|
|
#define rcu_can_boost() 0
|
2011-04-18 00:45:23 -06:00
|
|
|
#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
|
2010-09-02 17:16:14 -06:00
|
|
|
|
2011-11-04 12:44:12 -06:00
|
|
|
static unsigned long shutdown_time; /* jiffies to system shutdown. */
|
2010-09-02 17:16:14 -06:00
|
|
|
static unsigned long boost_starttime; /* jiffies of next boost test start. */
|
|
|
|
DEFINE_MUTEX(boost_mutex); /* protect setting boost_starttime */
|
|
|
|
/* and boost task create/destroy. */
|
|
|
|
|
2009-01-07 15:33:30 -07:00
|
|
|
/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
|
|
|
|
|
|
|
|
#define FULLSTOP_DONTSTOP 0 /* Normal operation. */
|
|
|
|
#define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */
|
|
|
|
#define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */
|
|
|
|
static int fullstop = FULLSTOP_RMMOD;
|
2010-09-19 22:06:14 -06:00
|
|
|
/*
|
|
|
|
* Protect fullstop transitions and spawning of kthreads.
|
|
|
|
*/
|
|
|
|
static DEFINE_MUTEX(fullstop_mutex);
|
2008-12-15 17:13:07 -07:00
|
|
|
|
2011-11-04 12:44:12 -06:00
|
|
|
/* Forward reference. */
|
|
|
|
static void rcu_torture_cleanup(void);
|
|
|
|
|
2008-12-15 17:13:07 -07:00
|
|
|
/*
|
2009-01-07 15:33:30 -07:00
|
|
|
* Detect and respond to a system shutdown.
|
2008-12-15 17:13:07 -07:00
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcutorture_shutdown_notify(struct notifier_block *unused1,
|
|
|
|
unsigned long unused2, void *unused3)
|
|
|
|
{
|
2009-01-04 19:28:27 -07:00
|
|
|
mutex_lock(&fullstop_mutex);
|
2009-01-07 15:33:30 -07:00
|
|
|
if (fullstop == FULLSTOP_DONTSTOP)
|
2009-01-04 19:28:27 -07:00
|
|
|
fullstop = FULLSTOP_SHUTDOWN;
|
2009-01-07 15:33:30 -07:00
|
|
|
else
|
|
|
|
printk(KERN_WARNING /* but going down anyway, so... */
|
|
|
|
"Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
|
2009-01-04 19:28:27 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2008-12-15 17:13:07 -07:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2009-01-07 15:33:30 -07:00
|
|
|
/*
|
|
|
|
* Absorb kthreads into a kernel function that won't return, so that
|
|
|
|
* they won't ever access module text or data again.
|
|
|
|
*/
|
|
|
|
static void rcutorture_shutdown_absorb(char *title)
|
|
|
|
{
|
|
|
|
if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
|
|
|
|
printk(KERN_NOTICE
|
|
|
|
"rcutorture thread %s parking due to system shutdown\n",
|
|
|
|
title);
|
|
|
|
schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
/*
|
|
|
|
* Allocate an element from the rcu_tortures pool.
|
|
|
|
*/
|
2006-01-08 02:02:17 -07:00
|
|
|
static struct rcu_torture *
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_torture_alloc(void)
|
|
|
|
{
|
|
|
|
struct list_head *p;
|
|
|
|
|
2006-01-25 11:50:12 -07:00
|
|
|
spin_lock_bh(&rcu_torture_lock);
|
2005-10-30 16:03:12 -07:00
|
|
|
if (list_empty(&rcu_torture_freelist)) {
|
|
|
|
atomic_inc(&n_rcu_torture_alloc_fail);
|
2006-01-25 11:50:12 -07:00
|
|
|
spin_unlock_bh(&rcu_torture_lock);
|
2005-10-30 16:03:12 -07:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
atomic_inc(&n_rcu_torture_alloc);
|
|
|
|
p = rcu_torture_freelist.next;
|
|
|
|
list_del_init(p);
|
2006-01-25 11:50:12 -07:00
|
|
|
spin_unlock_bh(&rcu_torture_lock);
|
2005-10-30 16:03:12 -07:00
|
|
|
return container_of(p, struct rcu_torture, rtort_free);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Free an element to the rcu_tortures pool.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
rcu_torture_free(struct rcu_torture *p)
|
|
|
|
{
|
|
|
|
atomic_inc(&n_rcu_torture_free);
|
2006-01-25 11:50:12 -07:00
|
|
|
spin_lock_bh(&rcu_torture_lock);
|
2005-10-30 16:03:12 -07:00
|
|
|
list_add_tail(&p->rtort_free, &rcu_torture_freelist);
|
2006-01-25 11:50:12 -07:00
|
|
|
spin_unlock_bh(&rcu_torture_lock);
|
2005-10-30 16:03:12 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
struct rcu_random_state {
|
|
|
|
unsigned long rrs_state;
|
2006-10-04 03:17:12 -06:00
|
|
|
long rrs_count;
|
2005-10-30 16:03:12 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
#define RCU_RANDOM_MULT 39916801 /* prime */
|
|
|
|
#define RCU_RANDOM_ADD 479001701 /* prime */
|
|
|
|
#define RCU_RANDOM_REFRESH 10000
|
|
|
|
|
|
|
|
#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Crude but fast random-number generator. Uses a linear congruential
|
2007-10-17 00:27:19 -06:00
|
|
|
* generator, with occasional help from cpu_clock().
|
2005-10-30 16:03:12 -07:00
|
|
|
*/
|
2006-10-04 03:17:12 -06:00
|
|
|
static unsigned long
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_random(struct rcu_random_state *rrsp)
|
|
|
|
{
|
|
|
|
if (--rrsp->rrs_count < 0) {
|
2010-05-25 02:48:51 -06:00
|
|
|
rrsp->rrs_state += (unsigned long)local_clock();
|
2005-10-30 16:03:12 -07:00
|
|
|
rrsp->rrs_count = RCU_RANDOM_REFRESH;
|
|
|
|
}
|
|
|
|
rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
|
|
|
|
return swahw32(rrsp->rrs_state);
|
|
|
|
}
|
|
|
|
|
2008-06-18 06:21:44 -06:00
|
|
|
static void
|
2009-01-07 15:33:30 -07:00
|
|
|
rcu_stutter_wait(char *title)
|
2008-06-18 06:21:44 -06:00
|
|
|
{
|
2009-01-07 15:33:30 -07:00
|
|
|
while (stutter_pause_test || !rcutorture_runnable) {
|
2008-06-22 14:06:38 -06:00
|
|
|
if (rcutorture_runnable)
|
|
|
|
schedule_timeout_interruptible(1);
|
|
|
|
else
|
2008-06-22 15:02:55 -06:00
|
|
|
schedule_timeout_interruptible(round_jiffies_relative(HZ));
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb(title);
|
2008-12-15 17:13:07 -07:00
|
|
|
}
|
2008-06-18 06:21:44 -06:00
|
|
|
}
|
|
|
|
|
2006-06-27 03:54:03 -06:00
|
|
|
/*
|
|
|
|
* Operations vector for selecting different types of tests.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct rcu_torture_ops {
|
|
|
|
void (*init)(void);
|
|
|
|
void (*cleanup)(void);
|
|
|
|
int (*readlock)(void);
|
2009-06-25 10:08:17 -06:00
|
|
|
void (*read_delay)(struct rcu_random_state *rrsp);
|
2006-06-27 03:54:03 -06:00
|
|
|
void (*readunlock)(int idx);
|
|
|
|
int (*completed)(void);
|
2009-06-25 10:08:17 -06:00
|
|
|
void (*deferred_free)(struct rcu_torture *p);
|
2006-10-04 03:17:13 -06:00
|
|
|
void (*sync)(void);
|
2008-05-12 13:21:05 -06:00
|
|
|
void (*cb_barrier)(void);
|
2010-01-04 16:09:10 -07:00
|
|
|
void (*fqs)(void);
|
2006-06-27 03:54:03 -06:00
|
|
|
int (*stats)(char *page);
|
2009-06-25 10:08:17 -06:00
|
|
|
int irq_capable;
|
2010-09-02 17:16:14 -06:00
|
|
|
int can_boost;
|
2006-06-27 03:54:03 -06:00
|
|
|
char *name;
|
|
|
|
};
|
2009-09-18 11:28:19 -06:00
|
|
|
|
|
|
|
static struct rcu_torture_ops *cur_ops;
|
2006-06-27 03:54:03 -06:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Definitions for rcu torture testing.
|
|
|
|
*/
|
|
|
|
|
2006-09-29 02:59:30 -06:00
|
|
|
static int rcu_torture_read_lock(void) __acquires(RCU)
|
2006-06-27 03:54:03 -06:00
|
|
|
{
|
|
|
|
rcu_read_lock();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:03 -06:00
|
|
|
static void rcu_read_delay(struct rcu_random_state *rrsp)
|
|
|
|
{
|
2009-09-08 16:54:35 -06:00
|
|
|
const unsigned long shortdelay_us = 200;
|
|
|
|
const unsigned long longdelay_ms = 50;
|
2006-10-04 03:17:03 -06:00
|
|
|
|
2009-09-08 16:54:35 -06:00
|
|
|
/* We want a short delay sometimes to make a reader delay the grace
|
|
|
|
* period, and we want a long delay occasionally to trigger
|
|
|
|
* force_quiescent_state. */
|
2006-10-04 03:17:03 -06:00
|
|
|
|
2009-09-08 16:54:35 -06:00
|
|
|
if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
|
|
|
|
mdelay(longdelay_ms);
|
|
|
|
if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
|
|
|
|
udelay(shortdelay_us);
|
2010-06-21 02:57:42 -06:00
|
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
|
|
|
|
preempt_schedule(); /* No QS if preempt_disable() in effect */
|
|
|
|
#endif
|
2006-10-04 03:17:03 -06:00
|
|
|
}
|
|
|
|
|
2006-09-29 02:59:30 -06:00
|
|
|
static void rcu_torture_read_unlock(int idx) __releases(RCU)
|
2006-06-27 03:54:03 -06:00
|
|
|
{
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rcu_torture_completed(void)
|
|
|
|
{
|
|
|
|
return rcu_batches_completed();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
rcu_torture_cb(struct rcu_head *p)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
|
|
|
|
|
2009-01-07 15:33:30 -07:00
|
|
|
if (fullstop != FULLSTOP_DONTSTOP) {
|
2006-06-27 03:54:03 -06:00
|
|
|
/* Test is ending, just drop callbacks on the floor. */
|
|
|
|
/* The next initialization will pick up the pieces. */
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
i = rp->rtort_pipe_count;
|
|
|
|
if (i > RCU_TORTURE_PIPE_LEN)
|
|
|
|
i = RCU_TORTURE_PIPE_LEN;
|
|
|
|
atomic_inc(&rcu_torture_wcount[i]);
|
|
|
|
if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
|
|
|
|
rp->rtort_mbtest = 0;
|
|
|
|
rcu_torture_free(rp);
|
|
|
|
} else
|
2009-06-25 10:08:17 -06:00
|
|
|
cur_ops->deferred_free(rp);
|
2006-06-27 03:54:03 -06:00
|
|
|
}
|
|
|
|
|
2009-12-02 13:10:15 -07:00
|
|
|
static int rcu_no_completed(void)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-06-27 03:54:03 -06:00
|
|
|
static void rcu_torture_deferred_free(struct rcu_torture *p)
|
|
|
|
{
|
|
|
|
call_rcu(&p->rtort_rcu, rcu_torture_cb);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rcu_torture_ops rcu_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = NULL,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay,
|
|
|
|
.readunlock = rcu_torture_read_unlock,
|
|
|
|
.completed = rcu_torture_completed,
|
|
|
|
.deferred_free = rcu_torture_deferred_free,
|
|
|
|
.sync = synchronize_rcu,
|
|
|
|
.cb_barrier = rcu_barrier,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
2009-09-18 11:28:19 -06:00
|
|
|
.irq_capable = 1,
|
2010-09-02 17:16:14 -06:00
|
|
|
.can_boost = rcu_can_boost(),
|
2009-09-18 11:28:19 -06:00
|
|
|
.name = "rcu"
|
2006-06-27 03:54:03 -06:00
|
|
|
};
|
|
|
|
|
2006-10-04 03:17:14 -06:00
|
|
|
static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct rcu_torture *rp;
|
|
|
|
struct rcu_torture *rp1;
|
|
|
|
|
|
|
|
cur_ops->sync();
|
|
|
|
list_add(&p->rtort_free, &rcu_torture_removed);
|
|
|
|
list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
|
|
|
|
i = rp->rtort_pipe_count;
|
|
|
|
if (i > RCU_TORTURE_PIPE_LEN)
|
|
|
|
i = RCU_TORTURE_PIPE_LEN;
|
|
|
|
atomic_inc(&rcu_torture_wcount[i]);
|
|
|
|
if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
|
|
|
|
rp->rtort_mbtest = 0;
|
|
|
|
list_del(&rp->rtort_free);
|
|
|
|
rcu_torture_free(rp);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rcu_sync_torture_init(void)
|
|
|
|
{
|
|
|
|
INIT_LIST_HEAD(&rcu_torture_removed);
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:15 -06:00
|
|
|
static struct rcu_torture_ops rcu_sync_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay,
|
|
|
|
.readunlock = rcu_torture_read_unlock,
|
|
|
|
.completed = rcu_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = synchronize_rcu,
|
|
|
|
.cb_barrier = NULL,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
2010-09-02 17:16:14 -06:00
|
|
|
.can_boost = rcu_can_boost(),
|
2009-06-25 10:08:17 -06:00
|
|
|
.name = "rcu_sync"
|
2006-10-04 03:17:15 -06:00
|
|
|
};
|
|
|
|
|
2009-12-02 13:10:15 -07:00
|
|
|
static struct rcu_torture_ops rcu_expedited_ops = {
|
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = rcu_torture_read_unlock,
|
|
|
|
.completed = rcu_no_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = synchronize_rcu_expedited,
|
|
|
|
.cb_barrier = NULL,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_force_quiescent_state,
|
2009-12-02 13:10:15 -07:00
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
2010-09-02 17:16:14 -06:00
|
|
|
.can_boost = rcu_can_boost(),
|
2009-12-02 13:10:15 -07:00
|
|
|
.name = "rcu_expedited"
|
|
|
|
};
|
|
|
|
|
2006-06-27 03:54:04 -06:00
|
|
|
/*
|
|
|
|
* Definitions for rcu_bh torture testing.
|
|
|
|
*/
|
|
|
|
|
2006-09-29 02:59:30 -06:00
|
|
|
static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
|
2006-06-27 03:54:04 -06:00
|
|
|
{
|
|
|
|
rcu_read_lock_bh();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-09-29 02:59:30 -06:00
|
|
|
static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
|
2006-06-27 03:54:04 -06:00
|
|
|
{
|
|
|
|
rcu_read_unlock_bh();
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rcu_bh_torture_completed(void)
|
|
|
|
{
|
|
|
|
return rcu_batches_completed_bh();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
|
|
|
|
{
|
|
|
|
call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rcu_torture_ops rcu_bh_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = NULL,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_bh_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = rcu_bh_torture_read_unlock,
|
|
|
|
.completed = rcu_bh_torture_completed,
|
|
|
|
.deferred_free = rcu_bh_torture_deferred_free,
|
2011-06-07 17:59:35 -06:00
|
|
|
.sync = synchronize_rcu_bh,
|
2009-06-25 10:08:17 -06:00
|
|
|
.cb_barrier = rcu_barrier_bh,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_bh_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
|
|
|
.name = "rcu_bh"
|
2006-06-27 03:54:04 -06:00
|
|
|
};
|
|
|
|
|
2006-10-04 03:17:16 -06:00
|
|
|
static struct rcu_torture_ops rcu_bh_sync_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_bh_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = rcu_bh_torture_read_unlock,
|
|
|
|
.completed = rcu_bh_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
2011-06-07 17:59:35 -06:00
|
|
|
.sync = synchronize_rcu_bh,
|
2009-06-25 10:08:17 -06:00
|
|
|
.cb_barrier = NULL,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_bh_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
|
|
|
.name = "rcu_bh_sync"
|
2006-10-04 03:17:16 -06:00
|
|
|
};
|
|
|
|
|
2011-06-07 17:59:35 -06:00
|
|
|
static struct rcu_torture_ops rcu_bh_expedited_ops = {
|
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = rcu_bh_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = rcu_bh_torture_read_unlock,
|
|
|
|
.completed = rcu_bh_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = synchronize_rcu_bh_expedited,
|
|
|
|
.cb_barrier = NULL,
|
|
|
|
.fqs = rcu_bh_force_quiescent_state,
|
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
|
|
|
.name = "rcu_bh_expedited"
|
|
|
|
};
|
|
|
|
|
2006-10-04 03:17:03 -06:00
|
|
|
/*
|
|
|
|
* Definitions for srcu torture testing.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static struct srcu_struct srcu_ctl;
|
|
|
|
|
|
|
|
static void srcu_torture_init(void)
|
|
|
|
{
|
|
|
|
init_srcu_struct(&srcu_ctl);
|
2006-10-04 03:17:14 -06:00
|
|
|
rcu_sync_torture_init();
|
2006-10-04 03:17:03 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
static void srcu_torture_cleanup(void)
|
|
|
|
{
|
|
|
|
synchronize_srcu(&srcu_ctl);
|
|
|
|
cleanup_srcu_struct(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
2006-12-06 21:40:19 -07:00
|
|
|
static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
|
2006-10-04 03:17:03 -06:00
|
|
|
{
|
|
|
|
return srcu_read_lock(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void srcu_read_delay(struct rcu_random_state *rrsp)
|
|
|
|
{
|
|
|
|
long delay;
|
|
|
|
const long uspertick = 1000000 / HZ;
|
|
|
|
const long longdelay = 10;
|
|
|
|
|
|
|
|
/* We want there to be long-running readers, but not all the time. */
|
|
|
|
|
|
|
|
delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
|
|
|
|
if (!delay)
|
|
|
|
schedule_timeout_interruptible(longdelay);
|
2010-06-21 02:57:42 -06:00
|
|
|
else
|
|
|
|
rcu_read_delay(rrsp);
|
2006-10-04 03:17:03 -06:00
|
|
|
}
|
|
|
|
|
2006-12-06 21:40:19 -07:00
|
|
|
static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
|
2006-10-04 03:17:03 -06:00
|
|
|
{
|
|
|
|
srcu_read_unlock(&srcu_ctl, idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int srcu_torture_completed(void)
|
|
|
|
{
|
|
|
|
return srcu_batches_completed(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:13 -06:00
|
|
|
static void srcu_torture_synchronize(void)
|
|
|
|
{
|
|
|
|
synchronize_srcu(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:03 -06:00
|
|
|
static int srcu_torture_stats(char *page)
|
|
|
|
{
|
|
|
|
int cnt = 0;
|
|
|
|
int cpu;
|
|
|
|
int idx = srcu_ctl.completed & 0x1;
|
|
|
|
|
|
|
|
cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
|
|
|
|
torture_type, TORTURE_FLAG, idx);
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
|
|
|
|
per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
|
|
|
|
per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
|
|
|
|
}
|
|
|
|
cnt += sprintf(&page[cnt], "\n");
|
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rcu_torture_ops srcu_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = srcu_torture_init,
|
|
|
|
.cleanup = srcu_torture_cleanup,
|
|
|
|
.readlock = srcu_torture_read_lock,
|
|
|
|
.read_delay = srcu_read_delay,
|
|
|
|
.readunlock = srcu_torture_read_unlock,
|
|
|
|
.completed = srcu_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = srcu_torture_synchronize,
|
|
|
|
.cb_barrier = NULL,
|
|
|
|
.stats = srcu_torture_stats,
|
|
|
|
.name = "srcu"
|
2006-10-04 03:17:03 -06:00
|
|
|
};
|
|
|
|
|
2011-12-05 16:36:31 -07:00
|
|
|
static int srcu_torture_read_lock_raw(void) __acquires(&srcu_ctl)
|
|
|
|
{
|
|
|
|
return srcu_read_lock_raw(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void srcu_torture_read_unlock_raw(int idx) __releases(&srcu_ctl)
|
|
|
|
{
|
|
|
|
srcu_read_unlock_raw(&srcu_ctl, idx);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rcu_torture_ops srcu_raw_ops = {
|
|
|
|
.init = srcu_torture_init,
|
|
|
|
.cleanup = srcu_torture_cleanup,
|
|
|
|
.readlock = srcu_torture_read_lock_raw,
|
|
|
|
.read_delay = srcu_read_delay,
|
|
|
|
.readunlock = srcu_torture_read_unlock_raw,
|
|
|
|
.completed = srcu_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = srcu_torture_synchronize,
|
|
|
|
.cb_barrier = NULL,
|
|
|
|
.stats = srcu_torture_stats,
|
|
|
|
.name = "srcu_raw"
|
|
|
|
};
|
|
|
|
|
2009-10-25 20:03:52 -06:00
|
|
|
static void srcu_torture_synchronize_expedited(void)
|
|
|
|
{
|
|
|
|
synchronize_srcu_expedited(&srcu_ctl);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct rcu_torture_ops srcu_expedited_ops = {
|
|
|
|
.init = srcu_torture_init,
|
|
|
|
.cleanup = srcu_torture_cleanup,
|
|
|
|
.readlock = srcu_torture_read_lock,
|
|
|
|
.read_delay = srcu_read_delay,
|
|
|
|
.readunlock = srcu_torture_read_unlock,
|
|
|
|
.completed = srcu_torture_completed,
|
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = srcu_torture_synchronize_expedited,
|
|
|
|
.cb_barrier = NULL,
|
|
|
|
.stats = srcu_torture_stats,
|
|
|
|
.name = "srcu_expedited"
|
|
|
|
};
|
|
|
|
|
2006-10-04 03:17:16 -06:00
|
|
|
/*
|
|
|
|
* Definitions for sched torture testing.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static int sched_torture_read_lock(void)
|
|
|
|
{
|
|
|
|
preempt_disable();
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void sched_torture_read_unlock(int idx)
|
|
|
|
{
|
|
|
|
preempt_enable();
|
|
|
|
}
|
|
|
|
|
2008-05-12 13:21:05 -06:00
|
|
|
static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
|
|
|
|
{
|
|
|
|
call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:16 -06:00
|
|
|
static struct rcu_torture_ops sched_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = sched_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = sched_torture_read_unlock,
|
2009-12-02 13:10:15 -07:00
|
|
|
.completed = rcu_no_completed,
|
2009-06-25 10:08:17 -06:00
|
|
|
.deferred_free = rcu_sched_torture_deferred_free,
|
2011-06-07 17:59:35 -06:00
|
|
|
.sync = synchronize_sched,
|
2009-06-25 10:08:17 -06:00
|
|
|
.cb_barrier = rcu_barrier_sched,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_sched_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
|
|
|
.irq_capable = 1,
|
|
|
|
.name = "sched"
|
2006-10-04 03:17:16 -06:00
|
|
|
};
|
|
|
|
|
2009-10-25 20:03:52 -06:00
|
|
|
static struct rcu_torture_ops sched_sync_ops = {
|
2009-06-25 10:08:17 -06:00
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = sched_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = sched_torture_read_unlock,
|
2009-12-02 13:10:15 -07:00
|
|
|
.completed = rcu_no_completed,
|
2009-06-25 10:08:17 -06:00
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
2011-06-07 17:59:35 -06:00
|
|
|
.sync = synchronize_sched,
|
2009-06-25 10:08:17 -06:00
|
|
|
.cb_barrier = NULL,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_sched_force_quiescent_state,
|
2009-06-25 10:08:17 -06:00
|
|
|
.stats = NULL,
|
|
|
|
.name = "sched_sync"
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct rcu_torture_ops sched_expedited_ops = {
|
|
|
|
.init = rcu_sync_torture_init,
|
|
|
|
.cleanup = NULL,
|
|
|
|
.readlock = sched_torture_read_lock,
|
|
|
|
.read_delay = rcu_read_delay, /* just reuse rcu's version. */
|
|
|
|
.readunlock = sched_torture_read_unlock,
|
2009-12-02 13:10:15 -07:00
|
|
|
.completed = rcu_no_completed,
|
2009-06-25 10:08:17 -06:00
|
|
|
.deferred_free = rcu_sync_torture_deferred_free,
|
|
|
|
.sync = synchronize_sched_expedited,
|
|
|
|
.cb_barrier = NULL,
|
2010-01-04 16:09:10 -07:00
|
|
|
.fqs = rcu_sched_force_quiescent_state,
|
2010-05-06 10:49:21 -06:00
|
|
|
.stats = NULL,
|
2009-06-25 10:08:17 -06:00
|
|
|
.irq_capable = 1,
|
|
|
|
.name = "sched_expedited"
|
2008-05-12 13:21:05 -06:00
|
|
|
};
|
|
|
|
|
2010-09-02 17:16:14 -06:00
|
|
|
/*
|
|
|
|
* RCU torture priority-boost testing. Runs one real-time thread per
|
|
|
|
* CPU for moderate bursts, repeatedly registering RCU callbacks and
|
|
|
|
* spinning waiting for them to be invoked. If a given callback takes
|
|
|
|
* too long to be invoked, we assume that priority inversion has occurred.
|
|
|
|
*/
|
|
|
|
|
|
|
|
struct rcu_boost_inflight {
|
|
|
|
struct rcu_head rcu;
|
|
|
|
int inflight;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void rcu_torture_boost_cb(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
struct rcu_boost_inflight *rbip =
|
|
|
|
container_of(head, struct rcu_boost_inflight, rcu);
|
|
|
|
|
|
|
|
smp_mb(); /* Ensure RCU-core accesses precede clearing ->inflight */
|
|
|
|
rbip->inflight = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rcu_torture_boost(void *arg)
|
|
|
|
{
|
|
|
|
unsigned long call_rcu_time;
|
|
|
|
unsigned long endtime;
|
|
|
|
unsigned long oldstarttime;
|
|
|
|
struct rcu_boost_inflight rbi = { .inflight = 0 };
|
|
|
|
struct sched_param sp;
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_boost started");
|
|
|
|
|
|
|
|
/* Set real-time priority. */
|
|
|
|
sp.sched_priority = 1;
|
|
|
|
if (sched_setscheduler(current, SCHED_FIFO, &sp) < 0) {
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_boost RT prio failed!");
|
|
|
|
n_rcu_torture_boost_rterror++;
|
|
|
|
}
|
|
|
|
|
2011-03-30 10:10:44 -06:00
|
|
|
init_rcu_head_on_stack(&rbi.rcu);
|
2010-09-02 17:16:14 -06:00
|
|
|
/* Each pass through the following loop does one boost-test cycle. */
|
|
|
|
do {
|
|
|
|
/* Wait for the next test interval. */
|
|
|
|
oldstarttime = boost_starttime;
|
2011-08-17 13:39:34 -06:00
|
|
|
while (ULONG_CMP_LT(jiffies, oldstarttime)) {
|
2010-09-02 17:16:14 -06:00
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
rcu_stutter_wait("rcu_torture_boost");
|
|
|
|
if (kthread_should_stop() ||
|
|
|
|
fullstop != FULLSTOP_DONTSTOP)
|
|
|
|
goto checkwait;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Do one boost-test interval. */
|
|
|
|
endtime = oldstarttime + test_boost_duration * HZ;
|
|
|
|
call_rcu_time = jiffies;
|
2011-08-17 13:39:34 -06:00
|
|
|
while (ULONG_CMP_LT(jiffies, endtime)) {
|
2010-09-02 17:16:14 -06:00
|
|
|
/* If we don't have a callback in flight, post one. */
|
|
|
|
if (!rbi.inflight) {
|
|
|
|
smp_mb(); /* RCU core before ->inflight = 1. */
|
|
|
|
rbi.inflight = 1;
|
|
|
|
call_rcu(&rbi.rcu, rcu_torture_boost_cb);
|
|
|
|
if (jiffies - call_rcu_time >
|
|
|
|
test_boost_duration * HZ - HZ / 2) {
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_boost boosting failed");
|
|
|
|
n_rcu_torture_boost_failure++;
|
|
|
|
}
|
|
|
|
call_rcu_time = jiffies;
|
|
|
|
}
|
|
|
|
cond_resched();
|
|
|
|
rcu_stutter_wait("rcu_torture_boost");
|
|
|
|
if (kthread_should_stop() ||
|
|
|
|
fullstop != FULLSTOP_DONTSTOP)
|
|
|
|
goto checkwait;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Set the start time of the next test interval.
|
|
|
|
* Yes, this is vulnerable to long delays, but such
|
|
|
|
* delays simply cause a false negative for the next
|
|
|
|
* interval. Besides, we are running at RT priority,
|
|
|
|
* so delays should be relatively rare.
|
|
|
|
*/
|
2011-08-18 10:30:32 -06:00
|
|
|
while (oldstarttime == boost_starttime &&
|
|
|
|
!kthread_should_stop()) {
|
2010-09-02 17:16:14 -06:00
|
|
|
if (mutex_trylock(&boost_mutex)) {
|
|
|
|
boost_starttime = jiffies +
|
|
|
|
test_boost_interval * HZ;
|
|
|
|
n_rcu_torture_boosts++;
|
|
|
|
mutex_unlock(&boost_mutex);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Go do the stutter. */
|
|
|
|
checkwait: rcu_stutter_wait("rcu_torture_boost");
|
|
|
|
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
|
|
|
|
|
|
|
/* Clean up and exit. */
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_boost task stopping");
|
|
|
|
rcutorture_shutdown_absorb("rcu_torture_boost");
|
|
|
|
while (!kthread_should_stop() || rbi.inflight)
|
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
smp_mb(); /* order accesses to ->inflight before stack-frame death. */
|
2011-06-21 02:48:03 -06:00
|
|
|
destroy_rcu_head_on_stack(&rbi.rcu);
|
2010-09-02 17:16:14 -06:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-01-04 16:09:10 -07:00
|
|
|
/*
|
|
|
|
* RCU torture force-quiescent-state kthread. Repeatedly induces
|
|
|
|
* bursts of calls to force_quiescent_state(), increasing the probability
|
|
|
|
* of occurrence of some important types of race conditions.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_fqs(void *arg)
|
|
|
|
{
|
|
|
|
unsigned long fqs_resume_time;
|
|
|
|
int fqs_burst_remaining;
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
|
|
|
|
do {
|
|
|
|
fqs_resume_time = jiffies + fqs_stutter * HZ;
|
2011-08-17 13:39:34 -06:00
|
|
|
while (ULONG_CMP_LT(jiffies, fqs_resume_time) &&
|
|
|
|
!kthread_should_stop()) {
|
2010-01-04 16:09:10 -07:00
|
|
|
schedule_timeout_interruptible(1);
|
|
|
|
}
|
|
|
|
fqs_burst_remaining = fqs_duration;
|
2011-08-17 13:39:34 -06:00
|
|
|
while (fqs_burst_remaining > 0 &&
|
|
|
|
!kthread_should_stop()) {
|
2010-01-04 16:09:10 -07:00
|
|
|
cur_ops->fqs();
|
|
|
|
udelay(fqs_holdoff);
|
|
|
|
fqs_burst_remaining -= fqs_holdoff;
|
|
|
|
}
|
|
|
|
rcu_stutter_wait("rcu_torture_fqs");
|
|
|
|
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
|
|
|
|
rcutorture_shutdown_absorb("rcu_torture_fqs");
|
|
|
|
while (!kthread_should_stop())
|
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
/*
|
|
|
|
* RCU torture writer kthread. Repeatedly substitutes a new structure
|
|
|
|
* for that pointed to by rcu_torture_current, freeing the old structure
|
|
|
|
* after a series of grace periods (the "pipeline").
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_writer(void *arg)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
long oldbatch = rcu_batches_completed();
|
|
|
|
struct rcu_torture *rp;
|
|
|
|
struct rcu_torture *old_rp;
|
|
|
|
static DEFINE_RCU_RANDOM(rand);
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
|
2005-11-13 17:07:22 -07:00
|
|
|
set_user_nice(current, 19);
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
do {
|
|
|
|
schedule_timeout_uninterruptible(1);
|
2009-09-18 11:28:19 -06:00
|
|
|
rp = rcu_torture_alloc();
|
|
|
|
if (rp == NULL)
|
2005-10-30 16:03:12 -07:00
|
|
|
continue;
|
|
|
|
rp->rtort_pipe_count = 0;
|
|
|
|
udelay(rcu_random(&rand) & 0x3ff);
|
2010-09-19 22:06:14 -06:00
|
|
|
old_rp = rcu_dereference_check(rcu_torture_current,
|
|
|
|
current == writer_task);
|
2005-11-18 02:10:50 -07:00
|
|
|
rp->rtort_mbtest = 1;
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_assign_pointer(rcu_torture_current, rp);
|
2009-09-23 10:50:43 -06:00
|
|
|
smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
|
2007-05-08 01:33:20 -06:00
|
|
|
if (old_rp) {
|
2005-10-30 16:03:12 -07:00
|
|
|
i = old_rp->rtort_pipe_count;
|
|
|
|
if (i > RCU_TORTURE_PIPE_LEN)
|
|
|
|
i = RCU_TORTURE_PIPE_LEN;
|
|
|
|
atomic_inc(&rcu_torture_wcount[i]);
|
|
|
|
old_rp->rtort_pipe_count++;
|
2009-06-25 10:08:17 -06:00
|
|
|
cur_ops->deferred_free(old_rp);
|
2005-10-30 16:03:12 -07:00
|
|
|
}
|
2011-04-03 22:33:51 -06:00
|
|
|
rcutorture_record_progress(++rcu_torture_current_version);
|
2006-06-27 03:54:03 -06:00
|
|
|
oldbatch = cur_ops->completed();
|
2009-01-07 15:33:30 -07:00
|
|
|
rcu_stutter_wait("rcu_torture_writer");
|
|
|
|
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_writer");
|
|
|
|
while (!kthread_should_stop())
|
2005-10-30 16:03:12 -07:00
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-10-04 03:17:13 -06:00
|
|
|
/*
|
|
|
|
* RCU torture fake writer kthread. Repeatedly calls sync, with a random
|
|
|
|
* delay between calls.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_fakewriter(void *arg)
|
|
|
|
{
|
|
|
|
DEFINE_RCU_RANDOM(rand);
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
|
|
|
|
set_user_nice(current, 19);
|
|
|
|
|
|
|
|
do {
|
|
|
|
schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
|
|
|
|
udelay(rcu_random(&rand) & 0x3ff);
|
|
|
|
cur_ops->sync();
|
2009-01-07 15:33:30 -07:00
|
|
|
rcu_stutter_wait("rcu_torture_fakewriter");
|
|
|
|
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
2006-10-04 03:17:13 -06:00
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_fakewriter");
|
|
|
|
while (!kthread_should_stop())
|
2006-10-04 03:17:13 -06:00
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-10-02 08:44:32 -06:00
|
|
|
void rcutorture_trace_dump(void)
|
|
|
|
{
|
|
|
|
static atomic_t beenhere = ATOMIC_INIT(0);
|
|
|
|
|
|
|
|
if (atomic_read(&beenhere))
|
|
|
|
return;
|
|
|
|
if (atomic_xchg(&beenhere, 1) != 0)
|
|
|
|
return;
|
|
|
|
do_trace_rcu_torture_read(cur_ops->name, (struct rcu_head *)~0UL);
|
|
|
|
ftrace_dump(DUMP_ALL);
|
|
|
|
}
|
|
|
|
|
2008-06-25 13:24:52 -06:00
|
|
|
/*
|
|
|
|
* RCU torture reader from timer handler. Dereferences rcu_torture_current,
|
|
|
|
* incrementing the corresponding element of the pipeline array. The
|
|
|
|
* counter in the element should never be greater than 1, otherwise, the
|
|
|
|
* RCU implementation is broken.
|
|
|
|
*/
|
|
|
|
static void rcu_torture_timer(unsigned long unused)
|
|
|
|
{
|
|
|
|
int idx;
|
|
|
|
int completed;
|
|
|
|
static DEFINE_RCU_RANDOM(rand);
|
|
|
|
static DEFINE_SPINLOCK(rand_lock);
|
|
|
|
struct rcu_torture *p;
|
|
|
|
int pipe_count;
|
|
|
|
|
|
|
|
idx = cur_ops->readlock();
|
|
|
|
completed = cur_ops->completed();
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-22 18:04:45 -07:00
|
|
|
p = rcu_dereference_check(rcu_torture_current,
|
|
|
|
rcu_read_lock_bh_held() ||
|
|
|
|
rcu_read_lock_sched_held() ||
|
|
|
|
srcu_read_lock_held(&srcu_ctl));
|
2011-10-02 08:44:32 -06:00
|
|
|
do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
|
2008-06-25 13:24:52 -06:00
|
|
|
if (p == NULL) {
|
|
|
|
/* Leave because rcu_torture_writer is not yet underway */
|
|
|
|
cur_ops->readunlock(idx);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
if (p->rtort_mbtest == 0)
|
|
|
|
atomic_inc(&n_rcu_torture_mberror);
|
|
|
|
spin_lock(&rand_lock);
|
2009-06-25 10:08:17 -06:00
|
|
|
cur_ops->read_delay(&rand);
|
2008-06-25 13:24:52 -06:00
|
|
|
n_rcu_torture_timers++;
|
|
|
|
spin_unlock(&rand_lock);
|
|
|
|
preempt_disable();
|
|
|
|
pipe_count = p->rtort_pipe_count;
|
|
|
|
if (pipe_count > RCU_TORTURE_PIPE_LEN) {
|
|
|
|
/* Should not happen, but... */
|
|
|
|
pipe_count = RCU_TORTURE_PIPE_LEN;
|
|
|
|
}
|
2011-10-02 08:44:32 -06:00
|
|
|
if (pipe_count > 1)
|
|
|
|
rcutorture_trace_dump();
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_inc(rcu_torture_count[pipe_count]);
|
2008-06-25 13:24:52 -06:00
|
|
|
completed = cur_ops->completed() - completed;
|
|
|
|
if (completed > RCU_TORTURE_PIPE_LEN) {
|
|
|
|
/* Should not happen, but... */
|
|
|
|
completed = RCU_TORTURE_PIPE_LEN;
|
|
|
|
}
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_inc(rcu_torture_batch[completed]);
|
2008-06-25 13:24:52 -06:00
|
|
|
preempt_enable();
|
|
|
|
cur_ops->readunlock(idx);
|
|
|
|
}
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
/*
|
|
|
|
* RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
|
|
|
|
* incrementing the corresponding element of the pipeline array. The
|
|
|
|
* counter in the element should never be greater than 1, otherwise, the
|
|
|
|
* RCU implementation is broken.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_reader(void *arg)
|
|
|
|
{
|
|
|
|
int completed;
|
2006-06-27 03:54:03 -06:00
|
|
|
int idx;
|
2005-10-30 16:03:12 -07:00
|
|
|
DEFINE_RCU_RANDOM(rand);
|
|
|
|
struct rcu_torture *p;
|
|
|
|
int pipe_count;
|
2008-06-25 13:24:52 -06:00
|
|
|
struct timer_list t;
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
|
2005-11-13 17:07:22 -07:00
|
|
|
set_user_nice(current, 19);
|
2009-06-25 10:08:17 -06:00
|
|
|
if (irqreader && cur_ops->irq_capable)
|
2008-06-25 13:24:52 -06:00
|
|
|
setup_timer_on_stack(&t, rcu_torture_timer, 0);
|
2005-11-13 17:07:22 -07:00
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
do {
|
2009-06-25 10:08:17 -06:00
|
|
|
if (irqreader && cur_ops->irq_capable) {
|
2008-06-25 13:24:52 -06:00
|
|
|
if (!timer_pending(&t))
|
2010-02-22 18:05:04 -07:00
|
|
|
mod_timer(&t, jiffies + 1);
|
2008-06-25 13:24:52 -06:00
|
|
|
}
|
2006-06-27 03:54:03 -06:00
|
|
|
idx = cur_ops->readlock();
|
|
|
|
completed = cur_ops->completed();
|
rcu: Introduce lockdep-based checking to RCU read-side primitives
Inspection is proving insufficient to catch all RCU misuses,
which is understandable given that rcu_dereference() might be
protected by any of four different flavors of RCU (RCU, RCU-bh,
RCU-sched, and SRCU), and might also/instead be protected by any
of a number of locking primitives. It is therefore time to
enlist the aid of lockdep.
This set of patches is inspired by earlier work by Peter
Zijlstra and Thomas Gleixner, and takes the following approach:
o Set up separate lockdep classes for RCU, RCU-bh, and RCU-sched.
o Set up separate lockdep classes for each instance of SRCU.
o Create primitives that check for being in an RCU read-side
critical section. These return exact answers if lockdep is
fully enabled, but if unsure, report being in an RCU read-side
critical section. (We want to avoid false positives!)
The primitives are:
For RCU: rcu_read_lock_held(void)
For RCU-bh: rcu_read_lock_bh_held(void)
For RCU-sched: rcu_read_lock_sched_held(void)
For SRCU: srcu_read_lock_held(struct srcu_struct *sp)
o Add rcu_dereference_check(), which takes a second argument
in which one places a boolean expression based on the above
primitives and/or lockdep_is_held().
o A new kernel configuration parameter, CONFIG_PROVE_RCU, enables
rcu_dereference_check(). This depends on CONFIG_PROVE_LOCKING,
and should be quite helpful during the transition period while
CONFIG_PROVE_RCU-unaware patches are in flight.
The existing rcu_dereference() primitive does no checking, but
upcoming patches will change that.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
LKML-Reference: <1266887105-1528-1-git-send-email-paulmck@linux.vnet.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2010-02-22 18:04:45 -07:00
|
|
|
p = rcu_dereference_check(rcu_torture_current,
|
|
|
|
rcu_read_lock_bh_held() ||
|
|
|
|
rcu_read_lock_sched_held() ||
|
|
|
|
srcu_read_lock_held(&srcu_ctl));
|
2011-10-02 08:44:32 -06:00
|
|
|
do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
|
2005-10-30 16:03:12 -07:00
|
|
|
if (p == NULL) {
|
|
|
|
/* Wait for rcu_torture_writer to get underway */
|
2006-06-27 03:54:03 -06:00
|
|
|
cur_ops->readunlock(idx);
|
2005-10-30 16:03:12 -07:00
|
|
|
schedule_timeout_interruptible(HZ);
|
|
|
|
continue;
|
|
|
|
}
|
2005-11-18 02:10:50 -07:00
|
|
|
if (p->rtort_mbtest == 0)
|
|
|
|
atomic_inc(&n_rcu_torture_mberror);
|
2009-06-25 10:08:17 -06:00
|
|
|
cur_ops->read_delay(&rand);
|
2005-10-30 16:03:12 -07:00
|
|
|
preempt_disable();
|
|
|
|
pipe_count = p->rtort_pipe_count;
|
|
|
|
if (pipe_count > RCU_TORTURE_PIPE_LEN) {
|
|
|
|
/* Should not happen, but... */
|
|
|
|
pipe_count = RCU_TORTURE_PIPE_LEN;
|
|
|
|
}
|
2011-10-02 08:44:32 -06:00
|
|
|
if (pipe_count > 1)
|
|
|
|
rcutorture_trace_dump();
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_inc(rcu_torture_count[pipe_count]);
|
2006-06-27 03:54:03 -06:00
|
|
|
completed = cur_ops->completed() - completed;
|
2005-10-30 16:03:12 -07:00
|
|
|
if (completed > RCU_TORTURE_PIPE_LEN) {
|
|
|
|
/* Should not happen, but... */
|
|
|
|
completed = RCU_TORTURE_PIPE_LEN;
|
|
|
|
}
|
2009-10-29 07:34:15 -06:00
|
|
|
__this_cpu_inc(rcu_torture_batch[completed]);
|
2005-10-30 16:03:12 -07:00
|
|
|
preempt_enable();
|
2006-06-27 03:54:03 -06:00
|
|
|
cur_ops->readunlock(idx);
|
2005-10-30 16:03:12 -07:00
|
|
|
schedule();
|
2009-01-07 15:33:30 -07:00
|
|
|
rcu_stutter_wait("rcu_torture_reader");
|
|
|
|
} while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_reader");
|
2009-06-25 10:08:17 -06:00
|
|
|
if (irqreader && cur_ops->irq_capable)
|
2008-06-25 13:24:52 -06:00
|
|
|
del_timer_sync(&t);
|
2009-01-07 15:33:30 -07:00
|
|
|
while (!kthread_should_stop())
|
2005-10-30 16:03:12 -07:00
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Create an RCU-torture statistics message in the specified buffer.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_printk(char *page)
|
|
|
|
{
|
|
|
|
int cnt = 0;
|
|
|
|
int cpu;
|
|
|
|
int i;
|
|
|
|
long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
|
|
|
|
long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
|
|
|
|
|
2006-03-28 02:56:37 -07:00
|
|
|
for_each_possible_cpu(cpu) {
|
2005-10-30 16:03:12 -07:00
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
|
|
|
|
pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
|
|
|
|
batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
|
|
|
|
if (pipesummary[i] != 0)
|
|
|
|
break;
|
|
|
|
}
|
2006-06-27 03:54:03 -06:00
|
|
|
cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt],
|
2011-04-03 22:33:51 -06:00
|
|
|
"rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d "
|
2011-02-21 14:31:55 -07:00
|
|
|
"rtmbe: %d rtbke: %ld rtbre: %ld "
|
2011-11-16 18:48:21 -07:00
|
|
|
"rtbf: %ld rtb: %ld nt: %ld "
|
|
|
|
"onoff: %ld/%ld:%ld/%ld",
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_torture_current,
|
|
|
|
rcu_torture_current_version,
|
|
|
|
list_empty(&rcu_torture_freelist),
|
|
|
|
atomic_read(&n_rcu_torture_alloc),
|
|
|
|
atomic_read(&n_rcu_torture_alloc_fail),
|
2005-11-18 02:10:50 -07:00
|
|
|
atomic_read(&n_rcu_torture_free),
|
2008-06-25 13:24:52 -06:00
|
|
|
atomic_read(&n_rcu_torture_mberror),
|
2010-09-02 17:16:14 -06:00
|
|
|
n_rcu_torture_boost_ktrerror,
|
|
|
|
n_rcu_torture_boost_rterror,
|
|
|
|
n_rcu_torture_boost_failure,
|
|
|
|
n_rcu_torture_boosts,
|
2011-11-16 18:48:21 -07:00
|
|
|
n_rcu_torture_timers,
|
|
|
|
n_online_successes,
|
|
|
|
n_online_attempts,
|
|
|
|
n_offline_successes,
|
|
|
|
n_offline_attempts);
|
2010-09-02 17:16:14 -06:00
|
|
|
if (atomic_read(&n_rcu_torture_mberror) != 0 ||
|
|
|
|
n_rcu_torture_boost_ktrerror != 0 ||
|
|
|
|
n_rcu_torture_boost_rterror != 0 ||
|
|
|
|
n_rcu_torture_boost_failure != 0)
|
2005-11-18 02:10:50 -07:00
|
|
|
cnt += sprintf(&page[cnt], " !!!");
|
2006-06-27 03:54:03 -06:00
|
|
|
cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
|
2005-11-18 02:10:50 -07:00
|
|
|
if (i > 1) {
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt], "!!! ");
|
2005-11-18 02:10:50 -07:00
|
|
|
atomic_inc(&n_rcu_torture_error);
|
2008-06-18 02:09:48 -06:00
|
|
|
WARN_ON_ONCE(1);
|
2005-11-18 02:10:50 -07:00
|
|
|
}
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt], "Reader Pipe: ");
|
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
|
|
|
|
cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
|
2006-06-27 03:54:03 -06:00
|
|
|
cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt], "Reader Batch: ");
|
2006-06-27 03:54:03 -06:00
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
|
2006-06-27 03:54:03 -06:00
|
|
|
cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
|
2005-10-30 16:03:12 -07:00
|
|
|
cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
|
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
|
|
|
|
cnt += sprintf(&page[cnt], " %d",
|
|
|
|
atomic_read(&rcu_torture_wcount[i]));
|
|
|
|
}
|
|
|
|
cnt += sprintf(&page[cnt], "\n");
|
2007-05-08 01:33:20 -06:00
|
|
|
if (cur_ops->stats)
|
2006-06-27 03:54:03 -06:00
|
|
|
cnt += cur_ops->stats(&page[cnt]);
|
2005-10-30 16:03:12 -07:00
|
|
|
return cnt;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Print torture statistics. Caller must ensure that there is only
|
|
|
|
* one call to this function at a given time!!! This is normally
|
|
|
|
* accomplished by relying on the module system to only have one copy
|
|
|
|
* of the module loaded, and then by giving the rcu_torture_stats
|
|
|
|
* kthread full control (or the init/cleanup functions when rcu_torture_stats
|
|
|
|
* thread is not running).
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
rcu_torture_stats_print(void)
|
|
|
|
{
|
|
|
|
int cnt;
|
|
|
|
|
|
|
|
cnt = rcu_torture_printk(printk_buf);
|
|
|
|
printk(KERN_ALERT "%s", printk_buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Periodically prints torture statistics, if periodic statistics printing
|
|
|
|
* was specified via the stat_interval module parameter.
|
|
|
|
*
|
|
|
|
* No need to worry about fullstop here, since this one doesn't reference
|
|
|
|
* volatile state or register callbacks.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_stats(void *arg)
|
|
|
|
{
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
|
|
|
|
do {
|
|
|
|
schedule_timeout_interruptible(stat_interval * HZ);
|
|
|
|
rcu_torture_stats_print();
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_stats");
|
|
|
|
} while (!kthread_should_stop());
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-01-08 02:03:42 -07:00
|
|
|
static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
|
|
|
|
|
|
|
|
/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
|
|
|
|
* is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
|
|
|
|
*/
|
2006-10-04 03:17:03 -06:00
|
|
|
static void rcu_torture_shuffle_tasks(void)
|
2006-01-08 02:03:42 -07:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2009-03-30 22:05:16 -06:00
|
|
|
cpumask_setall(shuffle_tmp_mask);
|
2008-01-25 13:08:02 -07:00
|
|
|
get_online_cpus();
|
2006-01-08 02:03:42 -07:00
|
|
|
|
|
|
|
/* No point in shuffling if there is only one online CPU (ex: UP) */
|
2009-01-07 15:33:30 -07:00
|
|
|
if (num_online_cpus() == 1) {
|
|
|
|
put_online_cpus();
|
|
|
|
return;
|
|
|
|
}
|
2006-01-08 02:03:42 -07:00
|
|
|
|
|
|
|
if (rcu_idle_cpu != -1)
|
2009-03-30 22:05:16 -06:00
|
|
|
cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
|
2009-03-30 22:05:16 -06:00
|
|
|
set_cpus_allowed_ptr(current, shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (reader_tasks) {
|
2006-01-08 02:03:42 -07:00
|
|
|
for (i = 0; i < nrealreaders; i++)
|
|
|
|
if (reader_tasks[i])
|
2008-04-04 19:11:06 -06:00
|
|
|
set_cpus_allowed_ptr(reader_tasks[i],
|
2009-03-30 22:05:16 -06:00
|
|
|
shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
}
|
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (fakewriter_tasks) {
|
2006-10-04 03:17:13 -06:00
|
|
|
for (i = 0; i < nfakewriters; i++)
|
|
|
|
if (fakewriter_tasks[i])
|
2008-04-04 19:11:06 -06:00
|
|
|
set_cpus_allowed_ptr(fakewriter_tasks[i],
|
2009-03-30 22:05:16 -06:00
|
|
|
shuffle_tmp_mask);
|
2006-10-04 03:17:13 -06:00
|
|
|
}
|
|
|
|
|
2006-01-08 02:03:42 -07:00
|
|
|
if (writer_task)
|
2009-03-30 22:05:16 -06:00
|
|
|
set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
|
|
|
|
if (stats_task)
|
2009-03-30 22:05:16 -06:00
|
|
|
set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
|
|
|
|
if (rcu_idle_cpu == -1)
|
|
|
|
rcu_idle_cpu = num_online_cpus() - 1;
|
|
|
|
else
|
|
|
|
rcu_idle_cpu--;
|
|
|
|
|
2008-01-25 13:08:02 -07:00
|
|
|
put_online_cpus();
|
2006-01-08 02:03:42 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
|
|
|
|
* system to become idle at a time and cut off its timer ticks. This is meant
|
|
|
|
* to test the support for such tickless idle CPU in RCU.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_shuffle(void *arg)
|
|
|
|
{
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
|
|
|
|
do {
|
|
|
|
schedule_timeout_interruptible(shuffle_interval * HZ);
|
|
|
|
rcu_torture_shuffle_tasks();
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_shuffle");
|
|
|
|
} while (!kthread_should_stop());
|
2006-01-08 02:03:42 -07:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2008-06-18 06:21:44 -06:00
|
|
|
/* Cause the rcutorture test to "stutter", starting and stopping all
|
|
|
|
* threads periodically.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_stutter(void *arg)
|
|
|
|
{
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
|
|
|
|
do {
|
|
|
|
schedule_timeout_interruptible(stutter * HZ);
|
|
|
|
stutter_pause_test = 1;
|
2009-01-07 15:33:30 -07:00
|
|
|
if (!kthread_should_stop())
|
2008-06-18 06:21:44 -06:00
|
|
|
schedule_timeout_interruptible(stutter * HZ);
|
|
|
|
stutter_pause_test = 0;
|
2009-01-07 15:33:30 -07:00
|
|
|
rcutorture_shutdown_absorb("rcu_torture_stutter");
|
|
|
|
} while (!kthread_should_stop());
|
2008-06-18 06:21:44 -06:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-03-24 04:15:58 -07:00
|
|
|
static inline void
|
2010-09-02 17:16:14 -06:00
|
|
|
rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag)
|
2006-03-24 04:15:58 -07:00
|
|
|
{
|
2006-10-04 03:17:13 -06:00
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"--- %s: nreaders=%d nfakewriters=%d "
|
2006-03-24 04:15:58 -07:00
|
|
|
"stat_interval=%d verbose=%d test_no_idle_hz=%d "
|
2010-01-04 16:09:10 -07:00
|
|
|
"shuffle_interval=%d stutter=%d irqreader=%d "
|
2010-09-02 17:16:14 -06:00
|
|
|
"fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
|
|
|
|
"test_boost=%d/%d test_boost_interval=%d "
|
2011-11-16 18:48:21 -07:00
|
|
|
"test_boost_duration=%d shutdown_secs=%d "
|
|
|
|
"onoff_interval=%d\n",
|
2006-10-04 03:17:13 -06:00
|
|
|
torture_type, tag, nrealreaders, nfakewriters,
|
2008-06-18 06:21:44 -06:00
|
|
|
stat_interval, verbose, test_no_idle_hz, shuffle_interval,
|
2010-09-02 17:16:14 -06:00
|
|
|
stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
|
|
|
|
test_boost, cur_ops->can_boost,
|
2011-11-16 18:48:21 -07:00
|
|
|
test_boost_interval, test_boost_duration, shutdown_secs,
|
|
|
|
onoff_interval);
|
2006-03-24 04:15:58 -07:00
|
|
|
}
|
|
|
|
|
2010-09-02 17:16:14 -06:00
|
|
|
static struct notifier_block rcutorture_shutdown_nb = {
|
2008-12-15 17:13:07 -07:00
|
|
|
.notifier_call = rcutorture_shutdown_notify,
|
|
|
|
};
|
|
|
|
|
2010-09-02 17:16:14 -06:00
|
|
|
static void rcutorture_booster_cleanup(int cpu)
|
|
|
|
{
|
|
|
|
struct task_struct *t;
|
|
|
|
|
|
|
|
if (boost_tasks[cpu] == NULL)
|
|
|
|
return;
|
|
|
|
mutex_lock(&boost_mutex);
|
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_boost task");
|
|
|
|
t = boost_tasks[cpu];
|
|
|
|
boost_tasks[cpu] = NULL;
|
|
|
|
mutex_unlock(&boost_mutex);
|
|
|
|
|
|
|
|
/* This must be outside of the mutex, otherwise deadlock! */
|
|
|
|
kthread_stop(t);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rcutorture_booster_init(int cpu)
|
|
|
|
{
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
if (boost_tasks[cpu] != NULL)
|
|
|
|
return 0; /* Already created, nothing more to do. */
|
|
|
|
|
|
|
|
/* Don't allow time recalculation while creating a new task. */
|
|
|
|
mutex_lock(&boost_mutex);
|
|
|
|
VERBOSE_PRINTK_STRING("Creating rcu_torture_boost task");
|
2011-06-16 16:53:18 -06:00
|
|
|
boost_tasks[cpu] = kthread_create_on_node(rcu_torture_boost, NULL,
|
|
|
|
cpu_to_node(cpu),
|
|
|
|
"rcu_torture_boost");
|
2010-09-02 17:16:14 -06:00
|
|
|
if (IS_ERR(boost_tasks[cpu])) {
|
|
|
|
retval = PTR_ERR(boost_tasks[cpu]);
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_boost task create failed");
|
|
|
|
n_rcu_torture_boost_ktrerror++;
|
|
|
|
boost_tasks[cpu] = NULL;
|
|
|
|
mutex_unlock(&boost_mutex);
|
|
|
|
return retval;
|
|
|
|
}
|
|
|
|
kthread_bind(boost_tasks[cpu], cpu);
|
|
|
|
wake_up_process(boost_tasks[cpu]);
|
|
|
|
mutex_unlock(&boost_mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-11-04 12:44:12 -06:00
|
|
|
/*
|
|
|
|
* Cause the rcutorture test to shutdown the system after the test has
|
|
|
|
* run for the time specified by the shutdown_secs module parameter.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_shutdown(void *arg)
|
|
|
|
{
|
|
|
|
long delta;
|
|
|
|
unsigned long jiffies_snap;
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_shutdown task started");
|
|
|
|
jiffies_snap = ACCESS_ONCE(jiffies);
|
|
|
|
while (ULONG_CMP_LT(jiffies_snap, shutdown_time) &&
|
|
|
|
!kthread_should_stop()) {
|
|
|
|
delta = shutdown_time - jiffies_snap;
|
|
|
|
if (verbose)
|
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"rcu_torture_shutdown task: %lu "
|
|
|
|
"jiffies remaining\n",
|
|
|
|
torture_type, delta);
|
|
|
|
schedule_timeout_interruptible(delta);
|
|
|
|
jiffies_snap = ACCESS_ONCE(jiffies);
|
|
|
|
}
|
2011-11-16 18:48:21 -07:00
|
|
|
if (kthread_should_stop()) {
|
2011-11-04 12:44:12 -06:00
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_shutdown task stopping");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* OK, shut down the system. */
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_shutdown task shutting down system");
|
|
|
|
shutdown_task = NULL; /* Avoid self-kill deadlock. */
|
|
|
|
rcu_torture_cleanup(); /* Get the success/failure message. */
|
|
|
|
kernel_power_off(); /* Shut down the system. */
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2011-11-16 18:48:21 -07:00
|
|
|
#ifdef CONFIG_HOTPLUG_CPU
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Execute random CPU-hotplug operations at the interval specified
|
|
|
|
* by the onoff_interval.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
rcu_torture_onoff(void *arg)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
int maxcpu = -1;
|
|
|
|
DEFINE_RCU_RANDOM(rand);
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_onoff task started");
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
maxcpu = cpu;
|
|
|
|
WARN_ON(maxcpu < 0);
|
|
|
|
while (!kthread_should_stop()) {
|
|
|
|
cpu = (rcu_random(&rand) >> 4) % (maxcpu + 1);
|
2011-12-03 16:09:28 -07:00
|
|
|
if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) {
|
2011-11-16 18:48:21 -07:00
|
|
|
if (verbose)
|
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"rcu_torture_onoff task: offlining %d\n",
|
|
|
|
torture_type, cpu);
|
|
|
|
n_offline_attempts++;
|
|
|
|
if (cpu_down(cpu) == 0) {
|
|
|
|
if (verbose)
|
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"rcu_torture_onoff task: "
|
|
|
|
"offlined %d\n",
|
|
|
|
torture_type, cpu);
|
|
|
|
n_offline_successes++;
|
|
|
|
}
|
2011-12-03 16:09:28 -07:00
|
|
|
} else if (cpu_is_hotpluggable(cpu)) {
|
2011-11-16 18:48:21 -07:00
|
|
|
if (verbose)
|
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"rcu_torture_onoff task: onlining %d\n",
|
|
|
|
torture_type, cpu);
|
|
|
|
n_online_attempts++;
|
|
|
|
if (cpu_up(cpu) == 0) {
|
|
|
|
if (verbose)
|
|
|
|
printk(KERN_ALERT "%s" TORTURE_FLAG
|
|
|
|
"rcu_torture_onoff task: "
|
|
|
|
"onlined %d\n",
|
|
|
|
torture_type, cpu);
|
|
|
|
n_online_successes++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
schedule_timeout_interruptible(onoff_interval * HZ);
|
|
|
|
}
|
|
|
|
VERBOSE_PRINTK_STRING("rcu_torture_onoff task stopping");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
rcu_torture_onoff_init(void)
|
|
|
|
{
|
|
|
|
if (onoff_interval <= 0)
|
|
|
|
return 0;
|
|
|
|
onoff_task = kthread_run(rcu_torture_onoff, NULL, "rcu_torture_onoff");
|
|
|
|
if (IS_ERR(onoff_task)) {
|
|
|
|
onoff_task = NULL;
|
|
|
|
return PTR_ERR(onoff_task);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rcu_torture_onoff_cleanup(void)
|
|
|
|
{
|
|
|
|
if (onoff_task == NULL)
|
|
|
|
return;
|
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_onoff task");
|
|
|
|
kthread_stop(onoff_task);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* #ifdef CONFIG_HOTPLUG_CPU */
|
|
|
|
|
|
|
|
static void
|
|
|
|
rcu_torture_onoff_init(void)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static void rcu_torture_onoff_cleanup(void)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
|
|
|
|
|
2010-09-02 17:16:14 -06:00
|
|
|
static int rcutorture_cpu_notify(struct notifier_block *self,
|
|
|
|
unsigned long action, void *hcpu)
|
|
|
|
{
|
|
|
|
long cpu = (long)hcpu;
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
case CPU_ONLINE:
|
|
|
|
case CPU_DOWN_FAILED:
|
|
|
|
(void)rcutorture_booster_init(cpu);
|
|
|
|
break;
|
|
|
|
case CPU_DOWN_PREPARE:
|
|
|
|
rcutorture_booster_cleanup(cpu);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
return NOTIFY_OK;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct notifier_block rcutorture_cpu_nb = {
|
|
|
|
.notifier_call = rcutorture_cpu_notify,
|
|
|
|
};
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
static void
|
|
|
|
rcu_torture_cleanup(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_lock(&fullstop_mutex);
|
2011-04-03 22:33:51 -06:00
|
|
|
rcutorture_record_test_transition();
|
2009-01-07 15:33:30 -07:00
|
|
|
if (fullstop == FULLSTOP_SHUTDOWN) {
|
|
|
|
printk(KERN_WARNING /* but going down anyway, so... */
|
|
|
|
"Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2009-01-07 15:33:30 -07:00
|
|
|
schedule_timeout_uninterruptible(10);
|
2008-12-15 17:13:07 -07:00
|
|
|
if (cur_ops->cb_barrier != NULL)
|
|
|
|
cur_ops->cb_barrier();
|
|
|
|
return;
|
|
|
|
}
|
2009-01-07 15:33:30 -07:00
|
|
|
fullstop = FULLSTOP_RMMOD;
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2010-09-02 17:16:14 -06:00
|
|
|
unregister_reboot_notifier(&rcutorture_shutdown_nb);
|
2008-06-18 06:21:44 -06:00
|
|
|
if (stutter_task) {
|
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
|
|
|
|
kthread_stop(stutter_task);
|
|
|
|
}
|
|
|
|
stutter_task = NULL;
|
2007-05-08 01:33:20 -06:00
|
|
|
if (shuffler_task) {
|
2006-01-08 02:03:42 -07:00
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
|
|
|
|
kthread_stop(shuffler_task);
|
2009-03-30 22:05:16 -06:00
|
|
|
free_cpumask_var(shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
}
|
|
|
|
shuffler_task = NULL;
|
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (writer_task) {
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
|
|
|
|
kthread_stop(writer_task);
|
|
|
|
}
|
|
|
|
writer_task = NULL;
|
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (reader_tasks) {
|
2005-10-30 16:03:12 -07:00
|
|
|
for (i = 0; i < nrealreaders; i++) {
|
2007-05-08 01:33:20 -06:00
|
|
|
if (reader_tasks[i]) {
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING(
|
|
|
|
"Stopping rcu_torture_reader task");
|
|
|
|
kthread_stop(reader_tasks[i]);
|
|
|
|
}
|
|
|
|
reader_tasks[i] = NULL;
|
|
|
|
}
|
|
|
|
kfree(reader_tasks);
|
|
|
|
reader_tasks = NULL;
|
|
|
|
}
|
|
|
|
rcu_torture_current = NULL;
|
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (fakewriter_tasks) {
|
2006-10-04 03:17:13 -06:00
|
|
|
for (i = 0; i < nfakewriters; i++) {
|
2007-05-08 01:33:20 -06:00
|
|
|
if (fakewriter_tasks[i]) {
|
2006-10-04 03:17:13 -06:00
|
|
|
VERBOSE_PRINTK_STRING(
|
|
|
|
"Stopping rcu_torture_fakewriter task");
|
|
|
|
kthread_stop(fakewriter_tasks[i]);
|
|
|
|
}
|
|
|
|
fakewriter_tasks[i] = NULL;
|
|
|
|
}
|
|
|
|
kfree(fakewriter_tasks);
|
|
|
|
fakewriter_tasks = NULL;
|
|
|
|
}
|
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (stats_task) {
|
2005-10-30 16:03:12 -07:00
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
|
|
|
|
kthread_stop(stats_task);
|
|
|
|
}
|
|
|
|
stats_task = NULL;
|
|
|
|
|
2010-01-04 16:09:10 -07:00
|
|
|
if (fqs_task) {
|
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
|
|
|
|
kthread_stop(fqs_task);
|
|
|
|
}
|
|
|
|
fqs_task = NULL;
|
2010-09-02 17:16:14 -06:00
|
|
|
if ((test_boost == 1 && cur_ops->can_boost) ||
|
|
|
|
test_boost == 2) {
|
|
|
|
unregister_cpu_notifier(&rcutorture_cpu_nb);
|
|
|
|
for_each_possible_cpu(i)
|
|
|
|
rcutorture_booster_cleanup(i);
|
|
|
|
}
|
2011-11-04 12:44:12 -06:00
|
|
|
if (shutdown_task != NULL) {
|
|
|
|
VERBOSE_PRINTK_STRING("Stopping rcu_torture_shutdown task");
|
|
|
|
kthread_stop(shutdown_task);
|
|
|
|
}
|
2011-11-16 18:48:21 -07:00
|
|
|
rcu_torture_onoff_cleanup();
|
2010-01-04 16:09:10 -07:00
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
/* Wait for all RCU callbacks to fire. */
|
2008-05-12 13:21:05 -06:00
|
|
|
|
|
|
|
if (cur_ops->cb_barrier != NULL)
|
|
|
|
cur_ops->cb_barrier();
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
|
2006-06-27 03:54:03 -06:00
|
|
|
|
2007-05-08 01:33:20 -06:00
|
|
|
if (cur_ops->cleanup)
|
2006-06-27 03:54:03 -06:00
|
|
|
cur_ops->cleanup();
|
2006-03-24 04:15:58 -07:00
|
|
|
if (atomic_read(&n_rcu_torture_error))
|
2010-09-02 17:16:14 -06:00
|
|
|
rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
|
2006-03-24 04:15:58 -07:00
|
|
|
else
|
2010-09-02 17:16:14 -06:00
|
|
|
rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
|
2005-10-30 16:03:12 -07:00
|
|
|
}
|
|
|
|
|
2007-05-08 01:25:24 -06:00
|
|
|
static int __init
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_torture_init(void)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int cpu;
|
|
|
|
int firsterr = 0;
|
2007-05-08 01:33:22 -06:00
|
|
|
static struct rcu_torture_ops *torture_ops[] =
|
2009-12-02 13:10:15 -07:00
|
|
|
{ &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
|
2011-06-07 17:59:35 -06:00
|
|
|
&rcu_bh_ops, &rcu_bh_sync_ops, &rcu_bh_expedited_ops,
|
2011-12-05 16:36:31 -07:00
|
|
|
&srcu_ops, &srcu_raw_ops, &srcu_expedited_ops,
|
2009-10-25 20:03:52 -06:00
|
|
|
&sched_ops, &sched_sync_ops, &sched_expedited_ops, };
|
2005-10-30 16:03:12 -07:00
|
|
|
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_lock(&fullstop_mutex);
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
/* Process args and tell the world that the torturer is on the job. */
|
2007-05-08 01:33:22 -06:00
|
|
|
for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
|
2006-06-27 03:54:03 -06:00
|
|
|
cur_ops = torture_ops[i];
|
2007-05-08 01:33:22 -06:00
|
|
|
if (strcmp(torture_type, cur_ops->name) == 0)
|
2006-06-27 03:54:03 -06:00
|
|
|
break;
|
|
|
|
}
|
2007-05-08 01:33:22 -06:00
|
|
|
if (i == ARRAY_SIZE(torture_ops)) {
|
2009-10-25 20:03:54 -06:00
|
|
|
printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n",
|
2006-06-27 03:54:03 -06:00
|
|
|
torture_type);
|
2009-10-25 20:03:54 -06:00
|
|
|
printk(KERN_ALERT "rcu-torture types:");
|
|
|
|
for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
|
|
|
|
printk(KERN_ALERT " %s", torture_ops[i]->name);
|
|
|
|
printk(KERN_ALERT "\n");
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2009-09-18 11:28:19 -06:00
|
|
|
return -EINVAL;
|
2006-06-27 03:54:03 -06:00
|
|
|
}
|
2010-01-04 16:09:10 -07:00
|
|
|
if (cur_ops->fqs == NULL && fqs_duration != 0) {
|
|
|
|
printk(KERN_ALERT "rcu-torture: ->fqs NULL and non-zero "
|
|
|
|
"fqs_duration, fqs disabled.\n");
|
|
|
|
fqs_duration = 0;
|
|
|
|
}
|
2007-05-08 01:33:20 -06:00
|
|
|
if (cur_ops->init)
|
2006-06-27 03:54:03 -06:00
|
|
|
cur_ops->init(); /* no "goto unwind" prior to this point!!! */
|
|
|
|
|
2005-10-30 16:03:12 -07:00
|
|
|
if (nreaders >= 0)
|
|
|
|
nrealreaders = nreaders;
|
|
|
|
else
|
|
|
|
nrealreaders = 2 * num_online_cpus();
|
2010-09-02 17:16:14 -06:00
|
|
|
rcu_torture_print_module_parms(cur_ops, "Start of test");
|
2009-01-07 15:33:30 -07:00
|
|
|
fullstop = FULLSTOP_DONTSTOP;
|
2005-10-30 16:03:12 -07:00
|
|
|
|
|
|
|
/* Set up the freelist. */
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&rcu_torture_freelist);
|
2007-05-08 01:33:14 -06:00
|
|
|
for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
|
2005-11-18 02:10:50 -07:00
|
|
|
rcu_tortures[i].rtort_mbtest = 0;
|
2005-10-30 16:03:12 -07:00
|
|
|
list_add_tail(&rcu_tortures[i].rtort_free,
|
|
|
|
&rcu_torture_freelist);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialize the statistics so that each run gets its own numbers. */
|
|
|
|
|
|
|
|
rcu_torture_current = NULL;
|
|
|
|
rcu_torture_current_version = 0;
|
|
|
|
atomic_set(&n_rcu_torture_alloc, 0);
|
|
|
|
atomic_set(&n_rcu_torture_alloc_fail, 0);
|
|
|
|
atomic_set(&n_rcu_torture_free, 0);
|
2005-11-18 02:10:50 -07:00
|
|
|
atomic_set(&n_rcu_torture_mberror, 0);
|
|
|
|
atomic_set(&n_rcu_torture_error, 0);
|
2010-09-02 17:16:14 -06:00
|
|
|
n_rcu_torture_boost_ktrerror = 0;
|
|
|
|
n_rcu_torture_boost_rterror = 0;
|
|
|
|
n_rcu_torture_boost_failure = 0;
|
|
|
|
n_rcu_torture_boosts = 0;
|
2005-10-30 16:03:12 -07:00
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
|
|
|
|
atomic_set(&rcu_torture_wcount[i], 0);
|
2006-03-28 02:56:37 -07:00
|
|
|
for_each_possible_cpu(cpu) {
|
2005-10-30 16:03:12 -07:00
|
|
|
for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
|
|
|
|
per_cpu(rcu_torture_count, cpu)[i] = 0;
|
|
|
|
per_cpu(rcu_torture_batch, cpu)[i] = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Start up the kthreads. */
|
|
|
|
|
|
|
|
VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
|
|
|
|
writer_task = kthread_run(rcu_torture_writer, NULL,
|
|
|
|
"rcu_torture_writer");
|
|
|
|
if (IS_ERR(writer_task)) {
|
|
|
|
firsterr = PTR_ERR(writer_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
|
|
|
|
writer_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
2006-10-04 03:17:13 -06:00
|
|
|
fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
|
2009-09-18 11:28:19 -06:00
|
|
|
GFP_KERNEL);
|
2006-10-04 03:17:13 -06:00
|
|
|
if (fakewriter_tasks == NULL) {
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("out of memory");
|
|
|
|
firsterr = -ENOMEM;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
for (i = 0; i < nfakewriters; i++) {
|
|
|
|
VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
|
|
|
|
fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
|
2009-09-18 11:28:19 -06:00
|
|
|
"rcu_torture_fakewriter");
|
2006-10-04 03:17:13 -06:00
|
|
|
if (IS_ERR(fakewriter_tasks[i])) {
|
|
|
|
firsterr = PTR_ERR(fakewriter_tasks[i]);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
|
|
|
|
fakewriter_tasks[i] = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2006-10-04 03:17:11 -06:00
|
|
|
reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
|
2005-10-30 16:03:12 -07:00
|
|
|
GFP_KERNEL);
|
|
|
|
if (reader_tasks == NULL) {
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("out of memory");
|
|
|
|
firsterr = -ENOMEM;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
for (i = 0; i < nrealreaders; i++) {
|
|
|
|
VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
|
|
|
|
reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
|
|
|
|
"rcu_torture_reader");
|
|
|
|
if (IS_ERR(reader_tasks[i])) {
|
|
|
|
firsterr = PTR_ERR(reader_tasks[i]);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
|
|
|
|
reader_tasks[i] = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (stat_interval > 0) {
|
|
|
|
VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
|
|
|
|
stats_task = kthread_run(rcu_torture_stats, NULL,
|
|
|
|
"rcu_torture_stats");
|
|
|
|
if (IS_ERR(stats_task)) {
|
|
|
|
firsterr = PTR_ERR(stats_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
|
|
|
|
stats_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2006-01-08 02:03:42 -07:00
|
|
|
if (test_no_idle_hz) {
|
|
|
|
rcu_idle_cpu = num_online_cpus() - 1;
|
2009-03-30 22:05:16 -06:00
|
|
|
|
|
|
|
if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
|
|
|
|
firsterr = -ENOMEM;
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
|
2006-01-08 02:03:42 -07:00
|
|
|
/* Create the shuffler thread */
|
|
|
|
shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
|
|
|
|
"rcu_torture_shuffle");
|
|
|
|
if (IS_ERR(shuffler_task)) {
|
2009-03-30 22:05:16 -06:00
|
|
|
free_cpumask_var(shuffle_tmp_mask);
|
2006-01-08 02:03:42 -07:00
|
|
|
firsterr = PTR_ERR(shuffler_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
|
|
|
|
shuffler_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2008-06-18 06:21:44 -06:00
|
|
|
if (stutter < 0)
|
|
|
|
stutter = 0;
|
|
|
|
if (stutter) {
|
|
|
|
/* Create the stutter thread */
|
|
|
|
stutter_task = kthread_run(rcu_torture_stutter, NULL,
|
|
|
|
"rcu_torture_stutter");
|
|
|
|
if (IS_ERR(stutter_task)) {
|
|
|
|
firsterr = PTR_ERR(stutter_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
|
|
|
|
stutter_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2010-01-04 16:09:10 -07:00
|
|
|
if (fqs_duration < 0)
|
|
|
|
fqs_duration = 0;
|
|
|
|
if (fqs_duration) {
|
|
|
|
/* Create the stutter thread */
|
|
|
|
fqs_task = kthread_run(rcu_torture_fqs, NULL,
|
|
|
|
"rcu_torture_fqs");
|
|
|
|
if (IS_ERR(fqs_task)) {
|
|
|
|
firsterr = PTR_ERR(fqs_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
|
|
|
|
fqs_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2010-09-02 17:16:14 -06:00
|
|
|
if (test_boost_interval < 1)
|
|
|
|
test_boost_interval = 1;
|
|
|
|
if (test_boost_duration < 2)
|
|
|
|
test_boost_duration = 2;
|
|
|
|
if ((test_boost == 1 && cur_ops->can_boost) ||
|
|
|
|
test_boost == 2) {
|
|
|
|
int retval;
|
|
|
|
|
|
|
|
boost_starttime = jiffies + test_boost_interval * HZ;
|
|
|
|
register_cpu_notifier(&rcutorture_cpu_nb);
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
if (cpu_is_offline(i))
|
|
|
|
continue; /* Heuristic: CPU can go offline. */
|
|
|
|
retval = rcutorture_booster_init(i);
|
|
|
|
if (retval < 0) {
|
|
|
|
firsterr = retval;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-11-04 12:44:12 -06:00
|
|
|
if (shutdown_secs > 0) {
|
|
|
|
shutdown_time = jiffies + shutdown_secs * HZ;
|
|
|
|
shutdown_task = kthread_run(rcu_torture_shutdown, NULL,
|
|
|
|
"rcu_torture_shutdown");
|
|
|
|
if (IS_ERR(shutdown_task)) {
|
|
|
|
firsterr = PTR_ERR(shutdown_task);
|
|
|
|
VERBOSE_PRINTK_ERRSTRING("Failed to create shutdown");
|
|
|
|
shutdown_task = NULL;
|
|
|
|
goto unwind;
|
|
|
|
}
|
|
|
|
}
|
2011-11-16 18:48:21 -07:00
|
|
|
rcu_torture_onoff_init();
|
2010-09-02 17:16:14 -06:00
|
|
|
register_reboot_notifier(&rcutorture_shutdown_nb);
|
2011-04-03 22:33:51 -06:00
|
|
|
rcutorture_record_test_transition();
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2005-10-30 16:03:12 -07:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
unwind:
|
2008-12-15 17:13:07 -07:00
|
|
|
mutex_unlock(&fullstop_mutex);
|
2005-10-30 16:03:12 -07:00
|
|
|
rcu_torture_cleanup();
|
|
|
|
return firsterr;
|
|
|
|
}
|
|
|
|
|
|
|
|
module_init(rcu_torture_init);
|
|
|
|
module_exit(rcu_torture_cleanup);
|