2006-05-23 18:18:44 -06:00
|
|
|
/*
|
|
|
|
* Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
|
|
* under the terms of the GNU General Public License as published by the Free
|
|
|
|
* Software Foundation; either version 2 of the License, or (at your option)
|
|
|
|
* any later version.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
* more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License along with
|
|
|
|
* this program; if not, write to the Free Software Foundation, Inc., 59
|
|
|
|
* Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
*
|
|
|
|
* The full GNU General Public License is included in this distribution in the
|
|
|
|
* file called COPYING.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This code implements the DMA subsystem. It provides a HW-neutral interface
|
|
|
|
* for other kernel code to use asynchronous memory copy capabilities,
|
|
|
|
* if present, and allows different HW DMA drivers to register as providing
|
|
|
|
* this capability.
|
|
|
|
*
|
|
|
|
* Due to the fact we are accelerating what is already a relatively fast
|
|
|
|
* operation, the code goes to great lengths to avoid additional overhead,
|
|
|
|
* such as locking.
|
|
|
|
*
|
|
|
|
* LOCKING:
|
|
|
|
*
|
2009-01-06 11:38:17 -07:00
|
|
|
* The subsystem keeps a global list of dma_device structs it is protected by a
|
|
|
|
* mutex, dma_list_mutex.
|
2006-05-23 18:18:44 -06:00
|
|
|
*
|
2009-01-06 11:38:18 -07:00
|
|
|
* A subsystem can get access to a channel by calling dmaengine_get() followed
|
|
|
|
* by dma_find_channel(), or if it has need for an exclusive channel it can call
|
|
|
|
* dma_request_channel(). Once a channel is allocated a reference is taken
|
|
|
|
* against its corresponding driver to disable removal.
|
|
|
|
*
|
2006-05-23 18:18:44 -06:00
|
|
|
* Each device has a channels list, which runs unlocked but is never modified
|
|
|
|
* once the device is registered, it's just setup by the driver.
|
|
|
|
*
|
2009-01-06 11:38:18 -07:00
|
|
|
* See Documentation/dmaengine.txt for more details
|
2006-05-23 18:18:44 -06:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/module.h>
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
#include <linux/mm.h>
|
2006-05-23 18:18:44 -06:00
|
|
|
#include <linux/device.h>
|
|
|
|
#include <linux/dmaengine.h>
|
|
|
|
#include <linux/hardirq.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/mutex.h>
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
#include <linux/jiffies.h>
|
2009-01-06 11:38:14 -07:00
|
|
|
#include <linux/rculist.h>
|
2009-01-06 11:38:21 -07:00
|
|
|
#include <linux/idr.h>
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
static DEFINE_MUTEX(dma_list_mutex);
|
|
|
|
static LIST_HEAD(dma_device_list);
|
2009-01-06 11:38:14 -07:00
|
|
|
static long dmaengine_ref_count;
|
2009-01-06 11:38:21 -07:00
|
|
|
static struct idr dma_idr;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
/* --- sysfs implementation --- */
|
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
/**
|
|
|
|
* dev_to_dma_chan - convert a device pointer to the its sysfs container object
|
|
|
|
* @dev - device node
|
|
|
|
*
|
|
|
|
* Must be called under dma_list_mutex
|
|
|
|
*/
|
|
|
|
static struct dma_chan *dev_to_dma_chan(struct device *dev)
|
|
|
|
{
|
|
|
|
struct dma_chan_dev *chan_dev;
|
|
|
|
|
|
|
|
chan_dev = container_of(dev, typeof(*chan_dev), device);
|
|
|
|
return chan_dev->chan;
|
|
|
|
}
|
|
|
|
|
2007-09-24 18:03:03 -06:00
|
|
|
static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf)
|
2006-05-23 18:18:44 -06:00
|
|
|
{
|
2009-01-06 11:38:21 -07:00
|
|
|
struct dma_chan *chan;
|
2006-05-23 18:18:44 -06:00
|
|
|
unsigned long count = 0;
|
|
|
|
int i;
|
2009-01-06 11:38:21 -07:00
|
|
|
int err;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan) {
|
|
|
|
for_each_possible_cpu(i)
|
|
|
|
count += per_cpu_ptr(chan->local, i)->memcpy_count;
|
|
|
|
err = sprintf(buf, "%lu\n", count);
|
|
|
|
} else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
return err;
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
|
|
|
|
2007-09-24 18:03:03 -06:00
|
|
|
static ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr,
|
|
|
|
char *buf)
|
2006-05-23 18:18:44 -06:00
|
|
|
{
|
2009-01-06 11:38:21 -07:00
|
|
|
struct dma_chan *chan;
|
2006-05-23 18:18:44 -06:00
|
|
|
unsigned long count = 0;
|
|
|
|
int i;
|
2009-01-06 11:38:21 -07:00
|
|
|
int err;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan) {
|
|
|
|
for_each_possible_cpu(i)
|
|
|
|
count += per_cpu_ptr(chan->local, i)->bytes_transferred;
|
|
|
|
err = sprintf(buf, "%lu\n", count);
|
|
|
|
} else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
return err;
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
|
|
|
|
2007-09-24 18:03:03 -06:00
|
|
|
static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf)
|
2006-05-23 18:18:44 -06:00
|
|
|
{
|
2009-01-06 11:38:21 -07:00
|
|
|
struct dma_chan *chan;
|
|
|
|
int err;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan = dev_to_dma_chan(dev);
|
|
|
|
if (chan)
|
|
|
|
err = sprintf(buf, "%d\n", chan->client_count);
|
|
|
|
else
|
|
|
|
err = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
return err;
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
|
|
|
|
2007-09-24 18:03:03 -06:00
|
|
|
static struct device_attribute dma_attrs[] = {
|
2006-05-23 18:18:44 -06:00
|
|
|
__ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL),
|
|
|
|
__ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL),
|
|
|
|
__ATTR(in_use, S_IRUGO, show_in_use, NULL),
|
|
|
|
__ATTR_NULL
|
|
|
|
};
|
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
static void chan_dev_release(struct device *dev)
|
|
|
|
{
|
|
|
|
struct dma_chan_dev *chan_dev;
|
|
|
|
|
|
|
|
chan_dev = container_of(dev, typeof(*chan_dev), device);
|
2009-01-06 11:38:21 -07:00
|
|
|
if (atomic_dec_and_test(chan_dev->idr_ref)) {
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
idr_remove(&dma_idr, chan_dev->dev_id);
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
kfree(chan_dev->idr_ref);
|
|
|
|
}
|
2009-01-06 11:38:21 -07:00
|
|
|
kfree(chan_dev);
|
|
|
|
}
|
|
|
|
|
2006-05-23 18:18:44 -06:00
|
|
|
static struct class dma_devclass = {
|
2007-09-24 18:03:03 -06:00
|
|
|
.name = "dma",
|
|
|
|
.dev_attrs = dma_attrs,
|
2009-01-06 11:38:21 -07:00
|
|
|
.dev_release = chan_dev_release,
|
2006-05-23 18:18:44 -06:00
|
|
|
};
|
|
|
|
|
|
|
|
/* --- client and device registration --- */
|
|
|
|
|
2009-01-06 11:38:15 -07:00
|
|
|
#define dma_device_satisfies_mask(device, mask) \
|
|
|
|
__dma_device_satisfies_mask((device), &(mask))
|
2007-07-09 12:56:42 -06:00
|
|
|
static int
|
2009-01-06 11:38:15 -07:00
|
|
|
__dma_device_satisfies_mask(struct dma_device *device, dma_cap_mask_t *want)
|
2007-07-09 12:56:42 -06:00
|
|
|
{
|
|
|
|
dma_cap_mask_t has;
|
|
|
|
|
2009-01-06 11:38:15 -07:00
|
|
|
bitmap_and(has.bits, want->bits, device->cap_mask.bits,
|
2007-07-09 12:56:42 -06:00
|
|
|
DMA_TX_TYPE_END);
|
|
|
|
return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
|
|
|
|
}
|
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
static struct module *dma_chan_to_owner(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
return chan->device->dev->driver->owner;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* balance_ref_count - catch up the channel reference count
|
|
|
|
* @chan - channel to balance ->client_count versus dmaengine_ref_count
|
|
|
|
*
|
|
|
|
* balance_ref_count must be called under dma_list_mutex
|
|
|
|
*/
|
|
|
|
static void balance_ref_count(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
struct module *owner = dma_chan_to_owner(chan);
|
|
|
|
|
|
|
|
while (chan->client_count < dmaengine_ref_count) {
|
|
|
|
__module_get(owner);
|
|
|
|
chan->client_count++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_chan_get - try to grab a dma channel's parent driver module
|
|
|
|
* @chan - channel to grab
|
|
|
|
*
|
|
|
|
* Must be called under dma_list_mutex
|
|
|
|
*/
|
|
|
|
static int dma_chan_get(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
int err = -ENODEV;
|
|
|
|
struct module *owner = dma_chan_to_owner(chan);
|
|
|
|
|
|
|
|
if (chan->client_count) {
|
|
|
|
__module_get(owner);
|
|
|
|
err = 0;
|
|
|
|
} else if (try_module_get(owner))
|
|
|
|
err = 0;
|
|
|
|
|
|
|
|
if (err == 0)
|
|
|
|
chan->client_count++;
|
|
|
|
|
|
|
|
/* allocate upon first client reference */
|
|
|
|
if (chan->client_count == 1 && err == 0) {
|
2009-01-06 11:38:17 -07:00
|
|
|
int desc_cnt = chan->device->device_alloc_chan_resources(chan);
|
2009-01-06 11:38:14 -07:00
|
|
|
|
|
|
|
if (desc_cnt < 0) {
|
|
|
|
err = desc_cnt;
|
|
|
|
chan->client_count = 0;
|
|
|
|
module_put(owner);
|
2009-01-06 11:38:15 -07:00
|
|
|
} else if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
|
2009-01-06 11:38:14 -07:00
|
|
|
balance_ref_count(chan);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_chan_put - drop a reference to a dma channel's parent driver module
|
|
|
|
* @chan - channel to release
|
|
|
|
*
|
|
|
|
* Must be called under dma_list_mutex
|
|
|
|
*/
|
|
|
|
static void dma_chan_put(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
if (!chan->client_count)
|
|
|
|
return; /* this channel failed alloc_chan_resources */
|
|
|
|
chan->client_count--;
|
|
|
|
module_put(dma_chan_to_owner(chan));
|
|
|
|
if (chan->client_count == 0)
|
|
|
|
chan->device->device_free_chan_resources(chan);
|
|
|
|
}
|
|
|
|
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
|
|
|
|
{
|
|
|
|
enum dma_status status;
|
|
|
|
unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
|
|
|
|
|
|
|
|
dma_async_issue_pending(chan);
|
|
|
|
do {
|
|
|
|
status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
|
|
|
|
if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
|
|
|
|
printk(KERN_ERR "dma_sync_wait_timeout!\n");
|
|
|
|
return DMA_ERROR;
|
|
|
|
}
|
|
|
|
} while (status == DMA_IN_PROGRESS);
|
|
|
|
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_sync_wait);
|
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
/**
|
|
|
|
* dma_cap_mask_all - enable iteration over all operation types
|
|
|
|
*/
|
|
|
|
static dma_cap_mask_t dma_cap_mask_all;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_chan_tbl_ent - tracks channel allocations per core/operation
|
|
|
|
* @chan - associated channel for this entry
|
|
|
|
*/
|
|
|
|
struct dma_chan_tbl_ent {
|
|
|
|
struct dma_chan *chan;
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* channel_table - percpu lookup table for memory-to-memory offload providers
|
|
|
|
*/
|
|
|
|
static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END];
|
|
|
|
|
|
|
|
static int __init dma_channel_table_init(void)
|
|
|
|
{
|
|
|
|
enum dma_transaction_type cap;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
|
|
|
|
|
2009-01-06 11:38:15 -07:00
|
|
|
/* 'interrupt', 'private', and 'slave' are channel capabilities,
|
|
|
|
* but are not associated with an operation so they do not need
|
|
|
|
* an entry in the channel_table
|
2009-01-06 11:38:14 -07:00
|
|
|
*/
|
|
|
|
clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
|
2009-01-06 11:38:15 -07:00
|
|
|
clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
|
2009-01-06 11:38:14 -07:00
|
|
|
clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
|
|
|
|
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all) {
|
|
|
|
channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
|
|
|
|
if (!channel_table[cap]) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
pr_err("dmaengine: initialization failure\n");
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
if (channel_table[cap])
|
|
|
|
free_percpu(channel_table[cap]);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
2009-01-06 11:38:22 -07:00
|
|
|
arch_initcall(dma_channel_table_init);
|
2009-01-06 11:38:14 -07:00
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_find_channel - find a channel to carry out the operation
|
|
|
|
* @tx_type: transaction type
|
|
|
|
*/
|
|
|
|
struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
cpu = get_cpu();
|
|
|
|
chan = per_cpu_ptr(channel_table[tx_type], cpu)->chan;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_find_channel);
|
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
/**
|
|
|
|
* dma_issue_pending_all - flush all pending operations across all channels
|
|
|
|
*/
|
|
|
|
void dma_issue_pending_all(void)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2009-01-06 11:38:15 -07:00
|
|
|
list_for_each_entry_rcu(device, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 11:38:14 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
if (chan->client_count)
|
|
|
|
device->device_issue_pending(chan);
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
2009-01-06 11:38:14 -07:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_issue_pending_all);
|
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
/**
|
|
|
|
* nth_chan - returns the nth channel of the given capability
|
|
|
|
* @cap: capability to match
|
|
|
|
* @n: nth channel desired
|
|
|
|
*
|
|
|
|
* Defaults to returning the channel with the desired capability and the
|
|
|
|
* lowest reference count when 'n' cannot be satisfied. Must be called
|
|
|
|
* under dma_list_mutex.
|
|
|
|
*/
|
|
|
|
static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n)
|
|
|
|
{
|
|
|
|
struct dma_device *device;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
struct dma_chan *ret = NULL;
|
|
|
|
struct dma_chan *min = NULL;
|
|
|
|
|
|
|
|
list_for_each_entry(device, &dma_device_list, global_node) {
|
2009-01-06 11:38:15 -07:00
|
|
|
if (!dma_has_cap(cap, device->cap_mask) ||
|
|
|
|
dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
2009-01-06 11:38:14 -07:00
|
|
|
continue;
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
if (!chan->client_count)
|
|
|
|
continue;
|
|
|
|
if (!min)
|
|
|
|
min = chan;
|
|
|
|
else if (chan->table_count < min->table_count)
|
|
|
|
min = chan;
|
|
|
|
|
|
|
|
if (n-- == 0) {
|
|
|
|
ret = chan;
|
|
|
|
break; /* done */
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (ret)
|
|
|
|
break; /* done */
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!ret)
|
|
|
|
ret = min;
|
|
|
|
|
|
|
|
if (ret)
|
|
|
|
ret->table_count++;
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_channel_rebalance - redistribute the available channels
|
|
|
|
*
|
|
|
|
* Optimize for cpu isolation (each cpu gets a dedicated channel for an
|
|
|
|
* operation type) in the SMP case, and operation isolation (avoid
|
|
|
|
* multi-tasking channels) in the non-SMP case. Must be called under
|
|
|
|
* dma_list_mutex.
|
|
|
|
*/
|
|
|
|
static void dma_channel_rebalance(void)
|
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
struct dma_device *device;
|
|
|
|
int cpu;
|
|
|
|
int cap;
|
|
|
|
int n;
|
|
|
|
|
|
|
|
/* undo the last distribution */
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
|
|
|
|
|
2009-01-06 11:38:15 -07:00
|
|
|
list_for_each_entry(device, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 11:38:14 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
chan->table_count = 0;
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
2009-01-06 11:38:14 -07:00
|
|
|
|
|
|
|
/* don't populate the channel_table if no clients are available */
|
|
|
|
if (!dmaengine_ref_count)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* redistribute available channels */
|
|
|
|
n = 0;
|
|
|
|
for_each_dma_cap_mask(cap, dma_cap_mask_all)
|
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
if (num_possible_cpus() > 1)
|
|
|
|
chan = nth_chan(cap, n++);
|
|
|
|
else
|
|
|
|
chan = nth_chan(cap, -1);
|
|
|
|
|
|
|
|
per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
static struct dma_chan *private_candidate(dma_cap_mask_t *mask, struct dma_device *dev,
|
|
|
|
dma_filter_fn fn, void *fn_param)
|
2009-01-06 11:38:15 -07:00
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
if (!__dma_device_satisfies_mask(dev, mask)) {
|
|
|
|
pr_debug("%s: wrong capabilities\n", __func__);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
/* devices with multiple channels need special handling as we need to
|
|
|
|
* ensure that all channels are either private or public.
|
|
|
|
*/
|
|
|
|
if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
|
|
|
|
list_for_each_entry(chan, &dev->channels, device_node) {
|
|
|
|
/* some channels are already publicly allocated */
|
|
|
|
if (chan->client_count)
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_for_each_entry(chan, &dev->channels, device_node) {
|
|
|
|
if (chan->client_count) {
|
|
|
|
pr_debug("%s: %s busy\n",
|
2009-01-06 11:38:21 -07:00
|
|
|
__func__, dma_chan_name(chan));
|
2009-01-06 11:38:15 -07:00
|
|
|
continue;
|
|
|
|
}
|
2009-01-06 11:38:21 -07:00
|
|
|
if (fn && !fn(chan, fn_param)) {
|
|
|
|
pr_debug("%s: %s filter said false\n",
|
|
|
|
__func__, dma_chan_name(chan));
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
return chan;
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
return NULL;
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_request_channel - try to allocate an exclusive channel
|
|
|
|
* @mask: capabilities that the channel must satisfy
|
|
|
|
* @fn: optional callback to disposition available channels
|
|
|
|
* @fn_param: opaque parameter to pass to dma_filter_fn
|
|
|
|
*/
|
|
|
|
struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param)
|
|
|
|
{
|
|
|
|
struct dma_device *device, *_d;
|
|
|
|
struct dma_chan *chan = NULL;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
/* Find a channel */
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
|
2009-01-06 11:38:21 -07:00
|
|
|
chan = private_candidate(mask, device, fn, fn_param);
|
|
|
|
if (chan) {
|
2009-01-06 11:38:15 -07:00
|
|
|
/* Found a suitable channel, try to grab, prep, and
|
|
|
|
* return it. We first set DMA_PRIVATE to disable
|
|
|
|
* balance_ref_count as this channel will not be
|
|
|
|
* published in the general-purpose allocator
|
|
|
|
*/
|
|
|
|
dma_cap_set(DMA_PRIVATE, device->cap_mask);
|
|
|
|
err = dma_chan_get(chan);
|
|
|
|
|
|
|
|
if (err == -ENODEV) {
|
|
|
|
pr_debug("%s: %s module removed\n", __func__,
|
2009-01-06 11:38:21 -07:00
|
|
|
dma_chan_name(chan));
|
2009-01-06 11:38:15 -07:00
|
|
|
list_del_rcu(&device->global_node);
|
|
|
|
} else if (err)
|
|
|
|
pr_err("dmaengine: failed to get %s: (%d)\n",
|
2009-01-06 11:38:21 -07:00
|
|
|
dma_chan_name(chan), err);
|
2009-01-06 11:38:15 -07:00
|
|
|
else
|
|
|
|
break;
|
2009-02-18 15:48:26 -07:00
|
|
|
chan->private = NULL;
|
2009-01-06 11:38:21 -07:00
|
|
|
chan = NULL;
|
|
|
|
}
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
pr_debug("%s: %s (%s)\n", __func__, chan ? "success" : "fail",
|
2009-01-06 11:38:21 -07:00
|
|
|
chan ? dma_chan_name(chan) : NULL);
|
2009-01-06 11:38:15 -07:00
|
|
|
|
|
|
|
return chan;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__dma_request_channel);
|
|
|
|
|
|
|
|
void dma_release_channel(struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
WARN_ONCE(chan->client_count != 1,
|
|
|
|
"chan reference count %d != 1\n", chan->client_count);
|
|
|
|
dma_chan_put(chan);
|
2009-02-18 15:48:26 -07:00
|
|
|
chan->private = NULL;
|
2009-01-06 11:38:15 -07:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_release_channel);
|
|
|
|
|
2007-07-09 12:56:42 -06:00
|
|
|
/**
|
2009-01-06 11:38:17 -07:00
|
|
|
* dmaengine_get - register interest in dma_channels
|
2007-07-09 12:56:42 -06:00
|
|
|
*/
|
2009-01-06 11:38:17 -07:00
|
|
|
void dmaengine_get(void)
|
2007-07-09 12:56:42 -06:00
|
|
|
{
|
2009-01-06 11:38:14 -07:00
|
|
|
struct dma_device *device, *_d;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
int err;
|
|
|
|
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 11:38:14 -07:00
|
|
|
dmaengine_ref_count++;
|
|
|
|
|
|
|
|
/* try to grab channels */
|
2009-01-06 11:38:15 -07:00
|
|
|
list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 11:38:14 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
err = dma_chan_get(chan);
|
|
|
|
if (err == -ENODEV) {
|
|
|
|
/* module removed before we could use it */
|
2009-01-06 11:38:14 -07:00
|
|
|
list_del_rcu(&device->global_node);
|
2009-01-06 11:38:14 -07:00
|
|
|
break;
|
|
|
|
} else if (err)
|
|
|
|
pr_err("dmaengine: failed to get %s: (%d)\n",
|
2009-01-06 11:38:21 -07:00
|
|
|
dma_chan_name(chan), err);
|
2009-01-06 11:38:14 -07:00
|
|
|
}
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
2009-01-06 11:38:14 -07:00
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
/* if this is the first reference and there were channels
|
|
|
|
* waiting we need to rebalance to get those channels
|
|
|
|
* incorporated into the channel table
|
|
|
|
*/
|
|
|
|
if (dmaengine_ref_count == 1)
|
|
|
|
dma_channel_rebalance();
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
2009-01-06 11:38:17 -07:00
|
|
|
EXPORT_SYMBOL(dmaengine_get);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
/**
|
2009-01-06 11:38:17 -07:00
|
|
|
* dmaengine_put - let dma drivers be removed when ref_count == 0
|
2006-05-23 18:18:44 -06:00
|
|
|
*/
|
2009-01-06 11:38:17 -07:00
|
|
|
void dmaengine_put(void)
|
2006-05-23 18:18:44 -06:00
|
|
|
{
|
2007-07-09 12:56:42 -06:00
|
|
|
struct dma_device *device;
|
2006-05-23 18:18:44 -06:00
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 11:38:14 -07:00
|
|
|
dmaengine_ref_count--;
|
|
|
|
BUG_ON(dmaengine_ref_count < 0);
|
|
|
|
/* drop channel references */
|
2009-01-06 11:38:15 -07:00
|
|
|
list_for_each_entry(device, &dma_device_list, global_node) {
|
|
|
|
if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
|
|
|
continue;
|
2009-01-06 11:38:14 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node)
|
|
|
|
dma_chan_put(chan);
|
2009-01-06 11:38:15 -07:00
|
|
|
}
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
}
|
2009-01-06 11:38:17 -07:00
|
|
|
EXPORT_SYMBOL(dmaengine_put);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-03-25 10:13:23 -06:00
|
|
|
static int get_dma_id(struct dma_device *device)
|
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
idr_retry:
|
|
|
|
if (!idr_pre_get(&dma_idr, GFP_KERNEL))
|
|
|
|
return -ENOMEM;
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
rc = idr_get_new(&dma_idr, NULL, &device->dev_id);
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
if (rc == -EAGAIN)
|
|
|
|
goto idr_retry;
|
|
|
|
else if (rc != 0)
|
|
|
|
return rc;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-05-23 18:18:44 -06:00
|
|
|
/**
|
2006-07-03 20:45:31 -06:00
|
|
|
* dma_async_device_register - registers DMA devices found
|
2006-05-23 18:18:44 -06:00
|
|
|
* @device: &dma_device
|
|
|
|
*/
|
|
|
|
int dma_async_device_register(struct dma_device *device)
|
|
|
|
{
|
2007-03-08 10:57:34 -07:00
|
|
|
int chancnt = 0, rc;
|
2006-05-23 18:18:44 -06:00
|
|
|
struct dma_chan* chan;
|
2009-01-06 11:38:21 -07:00
|
|
|
atomic_t *idr_ref;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
if (!device)
|
|
|
|
return -ENODEV;
|
|
|
|
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
/* validate device routines */
|
|
|
|
BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
|
|
|
|
!device->device_prep_dma_memcpy);
|
|
|
|
BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
|
|
|
|
!device->device_prep_dma_xor);
|
|
|
|
BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
|
|
|
|
!device->device_prep_dma_zero_sum);
|
|
|
|
BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
|
|
|
|
!device->device_prep_dma_memset);
|
2008-03-13 18:45:28 -06:00
|
|
|
BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
!device->device_prep_dma_interrupt);
|
dmaengine: Add slave DMA interface
This patch adds the necessary interfaces to the DMA Engine framework
to use functionality found on most embedded DMA controllers: DMA from
and to I/O registers with hardware handshaking.
In this context, hardware hanshaking means that the peripheral that
owns the I/O registers in question is able to tell the DMA controller
when more data is available for reading, or when there is room for
more data to be written. This usually happens internally on the chip,
but these signals may also be exported outside the chip for things
like IDE DMA, etc.
A new struct dma_slave is introduced. This contains information that
the DMA engine driver needs to set up slave transfers to and from a
slave device. Most engines supporting DMA slave transfers will want to
extend this structure with controller-specific parameters. This
additional information is usually passed from the platform/board code
through the client driver.
A "slave" pointer is added to the dma_client struct. This must point
to a valid dma_slave structure iff the DMA_SLAVE capability is
requested. The DMA engine driver may use this information in its
device_alloc_chan_resources hook to configure the DMA controller for
slave transfers from and to the given slave device.
A new operation for preparing slave DMA transfers is added to struct
dma_device. This takes a scatterlist and returns a single descriptor
representing the whole transfer.
Another new operation for terminating all pending transfers is added as
well. The latter is needed because there may be errors outside the scope
of the DMA Engine framework that may require DMA operations to be
terminated prematurely.
DMA Engine drivers may extend the dma_device, dma_chan and/or
dma_slave_descriptor structures to allow controller-specific
operations. The client driver can detect such extensions by looking at
the DMA Engine's struct device, or it can request a specific DMA
Engine device by setting the dma_dev field in struct dma_slave.
dmaslave interface changes since v4:
* Fix checkpatch errors
* Fix changelog (there are no slave descriptors anymore)
dmaslave interface changes since v3:
* Use dma_data_direction instead of a new enum
* Submit slave transfers as scatterlists
* Remove the DMA slave descriptor struct
dmaslave interface changes since v2:
* Add a dma_dev field to struct dma_slave. If set, the client can
only be bound to the DMA controller that corresponds to this
device. This allows controller-specific extensions of the
dma_slave structure; if the device matches, the controller may
safely assume its extensions are present.
* Move reg_width into struct dma_slave as there are currently no
users that need to be able to set the width on a per-transfer
basis.
dmaslave interface changes since v1:
* Drop the set_direction and set_width descriptor hooks. Pass the
direction and width to the prep function instead.
* Declare a dma_slave struct with fixed information about a slave,
i.e. register addresses, handshake interfaces and such.
* Add pointer to a dma_slave struct to dma_client. Can be NULL if
the DMA_SLAVE capability isn't requested.
* Drop the set_slave device hook since the alloc_chan_resources hook
now has enough information to set up the channel for slave
transfers.
Acked-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Haavard Skinnemoen <haavard.skinnemoen@atmel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
2008-07-08 12:59:35 -06:00
|
|
|
BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
|
|
|
|
!device->device_prep_slave_sg);
|
|
|
|
BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
|
|
|
|
!device->device_terminate_all);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
|
|
|
|
BUG_ON(!device->device_alloc_chan_resources);
|
|
|
|
BUG_ON(!device->device_free_chan_resources);
|
|
|
|
BUG_ON(!device->device_is_tx_complete);
|
|
|
|
BUG_ON(!device->device_issue_pending);
|
|
|
|
BUG_ON(!device->dev);
|
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
|
|
|
|
if (!idr_ref)
|
|
|
|
return -ENOMEM;
|
2009-03-25 10:13:23 -06:00
|
|
|
rc = get_dma_id(device);
|
|
|
|
if (rc != 0) {
|
|
|
|
kfree(idr_ref);
|
2009-01-06 11:38:21 -07:00
|
|
|
return rc;
|
2009-03-25 10:13:23 -06:00
|
|
|
}
|
|
|
|
|
|
|
|
atomic_set(idr_ref, 0);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
/* represent channels in sysfs. Probably want devs too */
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
2009-03-25 10:13:23 -06:00
|
|
|
rc = -ENOMEM;
|
2006-05-23 18:18:44 -06:00
|
|
|
chan->local = alloc_percpu(typeof(*chan->local));
|
|
|
|
if (chan->local == NULL)
|
2009-03-25 10:13:23 -06:00
|
|
|
goto err_out;
|
2009-01-06 11:38:21 -07:00
|
|
|
chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL);
|
|
|
|
if (chan->dev == NULL) {
|
|
|
|
free_percpu(chan->local);
|
2009-03-25 10:13:23 -06:00
|
|
|
chan->local = NULL;
|
|
|
|
goto err_out;
|
2009-01-06 11:38:21 -07:00
|
|
|
}
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
chan->chan_id = chancnt++;
|
2009-01-06 11:38:21 -07:00
|
|
|
chan->dev->device.class = &dma_devclass;
|
|
|
|
chan->dev->device.parent = device->dev;
|
|
|
|
chan->dev->chan = chan;
|
2009-01-06 11:38:21 -07:00
|
|
|
chan->dev->idr_ref = idr_ref;
|
|
|
|
chan->dev->dev_id = device->dev_id;
|
|
|
|
atomic_inc(idr_ref);
|
2009-01-06 11:38:21 -07:00
|
|
|
dev_set_name(&chan->dev->device, "dma%dchan%d",
|
2008-11-11 13:12:33 -07:00
|
|
|
device->dev_id, chan->chan_id);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:21 -07:00
|
|
|
rc = device_register(&chan->dev->device);
|
2007-03-08 10:57:34 -07:00
|
|
|
if (rc) {
|
|
|
|
free_percpu(chan->local);
|
|
|
|
chan->local = NULL;
|
2009-03-25 10:13:23 -06:00
|
|
|
kfree(chan->dev);
|
|
|
|
atomic_dec(idr_ref);
|
2007-03-08 10:57:34 -07:00
|
|
|
goto err_out;
|
|
|
|
}
|
2008-07-08 12:58:21 -06:00
|
|
|
chan->client_count = 0;
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
2009-01-06 11:38:15 -07:00
|
|
|
device->chancnt = chancnt;
|
2006-05-23 18:18:44 -06:00
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 11:38:15 -07:00
|
|
|
/* take references on public channels */
|
|
|
|
if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
|
2009-01-06 11:38:14 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
/* if clients are already waiting for channels we need
|
|
|
|
* to take references on their behalf
|
|
|
|
*/
|
|
|
|
if (dma_chan_get(chan) == -ENODEV) {
|
|
|
|
/* note we can only get here for the first
|
|
|
|
* channel as the remaining channels are
|
|
|
|
* guaranteed to get a reference
|
|
|
|
*/
|
|
|
|
rc = -ENODEV;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
goto err_out;
|
|
|
|
}
|
|
|
|
}
|
2009-01-06 11:38:14 -07:00
|
|
|
list_add_tail_rcu(&device->global_node, &dma_device_list);
|
2009-01-06 11:38:14 -07:00
|
|
|
dma_channel_rebalance();
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
return 0;
|
2007-03-08 10:57:34 -07:00
|
|
|
|
|
|
|
err_out:
|
2009-03-25 10:13:23 -06:00
|
|
|
/* if we never registered a channel just release the idr */
|
|
|
|
if (atomic_read(idr_ref) == 0) {
|
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
idr_remove(&dma_idr, device->dev_id);
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
kfree(idr_ref);
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
|
2007-03-08 10:57:34 -07:00
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
|
|
|
if (chan->local == NULL)
|
|
|
|
continue;
|
2009-01-06 11:38:21 -07:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan->dev->chan = NULL;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
device_unregister(&chan->dev->device);
|
2007-03-08 10:57:34 -07:00
|
|
|
free_percpu(chan->local);
|
|
|
|
}
|
|
|
|
return rc;
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
2007-03-16 15:38:05 -06:00
|
|
|
EXPORT_SYMBOL(dma_async_device_register);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2006-07-03 20:45:31 -06:00
|
|
|
/**
|
2009-01-06 11:38:14 -07:00
|
|
|
* dma_async_device_unregister - unregister a DMA device
|
2006-07-03 20:45:31 -06:00
|
|
|
* @device: &dma_device
|
2009-01-06 11:38:18 -07:00
|
|
|
*
|
|
|
|
* This routine is called by dma driver exit routines, dmaengine holds module
|
|
|
|
* references to prevent it being called while channels are in use.
|
2006-07-03 20:45:31 -06:00
|
|
|
*/
|
|
|
|
void dma_async_device_unregister(struct dma_device *device)
|
2006-05-23 18:18:44 -06:00
|
|
|
{
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
mutex_lock(&dma_list_mutex);
|
2009-01-06 11:38:14 -07:00
|
|
|
list_del_rcu(&device->global_node);
|
2009-01-06 11:38:14 -07:00
|
|
|
dma_channel_rebalance();
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
|
|
|
|
list_for_each_entry(chan, &device->channels, device_node) {
|
2009-01-06 11:38:14 -07:00
|
|
|
WARN_ONCE(chan->client_count,
|
|
|
|
"%s called while %d clients hold a reference\n",
|
|
|
|
__func__, chan->client_count);
|
2009-01-06 11:38:21 -07:00
|
|
|
mutex_lock(&dma_list_mutex);
|
|
|
|
chan->dev->chan = NULL;
|
|
|
|
mutex_unlock(&dma_list_mutex);
|
|
|
|
device_unregister(&chan->dev->device);
|
2006-05-23 18:18:44 -06:00
|
|
|
}
|
|
|
|
}
|
2007-03-16 15:38:05 -06:00
|
|
|
EXPORT_SYMBOL(dma_async_device_unregister);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
/**
|
|
|
|
* dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
|
|
|
|
* @chan: DMA channel to offload copy to
|
|
|
|
* @dest: destination address (virtual)
|
|
|
|
* @src: source address (virtual)
|
|
|
|
* @len: length
|
|
|
|
*
|
|
|
|
* Both @dest and @src must be mappable to a bus address according to the
|
|
|
|
* DMA mapping API rules for streaming mappings.
|
|
|
|
* Both @dest and @src must stay memory resident (kernel memory or locked
|
|
|
|
* user space pages).
|
|
|
|
*/
|
|
|
|
dma_cookie_t
|
|
|
|
dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
|
|
|
|
void *src, size_t len)
|
|
|
|
{
|
|
|
|
struct dma_device *dev = chan->device;
|
|
|
|
struct dma_async_tx_descriptor *tx;
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_addr_t dma_dest, dma_src;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
dma_cookie_t cookie;
|
|
|
|
int cpu;
|
|
|
|
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
|
|
|
|
dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
|
2008-04-17 21:17:26 -06:00
|
|
|
tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
|
|
|
|
DMA_CTRL_ACK);
|
2008-02-02 19:49:57 -07:00
|
|
|
|
|
|
|
if (!tx) {
|
|
|
|
dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
|
|
|
|
dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
return -ENOMEM;
|
2008-02-02 19:49:57 -07:00
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
|
|
|
|
tx->callback = NULL;
|
|
|
|
cookie = tx->tx_submit(tx);
|
|
|
|
|
|
|
|
cpu = get_cpu();
|
|
|
|
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
|
|
|
|
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return cookie;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_async_memcpy_buf_to_pg - offloaded copy from address to page
|
|
|
|
* @chan: DMA channel to offload copy to
|
|
|
|
* @page: destination page
|
|
|
|
* @offset: offset in page to copy to
|
|
|
|
* @kdata: source address (virtual)
|
|
|
|
* @len: length
|
|
|
|
*
|
|
|
|
* Both @page/@offset and @kdata must be mappable to a bus address according
|
|
|
|
* to the DMA mapping API rules for streaming mappings.
|
|
|
|
* Both @page/@offset and @kdata must stay memory resident (kernel memory or
|
|
|
|
* locked user space pages)
|
|
|
|
*/
|
|
|
|
dma_cookie_t
|
|
|
|
dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
|
|
|
|
unsigned int offset, void *kdata, size_t len)
|
|
|
|
{
|
|
|
|
struct dma_device *dev = chan->device;
|
|
|
|
struct dma_async_tx_descriptor *tx;
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_addr_t dma_dest, dma_src;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
dma_cookie_t cookie;
|
|
|
|
int cpu;
|
|
|
|
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
|
|
|
|
dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
|
2008-04-17 21:17:26 -06:00
|
|
|
tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
|
|
|
|
DMA_CTRL_ACK);
|
2008-02-02 19:49:57 -07:00
|
|
|
|
|
|
|
if (!tx) {
|
|
|
|
dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
|
|
|
|
dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
return -ENOMEM;
|
2008-02-02 19:49:57 -07:00
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
|
|
|
|
tx->callback = NULL;
|
|
|
|
cookie = tx->tx_submit(tx);
|
|
|
|
|
|
|
|
cpu = get_cpu();
|
|
|
|
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
|
|
|
|
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return cookie;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* dma_async_memcpy_pg_to_pg - offloaded copy from page to page
|
|
|
|
* @chan: DMA channel to offload copy to
|
|
|
|
* @dest_pg: destination page
|
|
|
|
* @dest_off: offset in page to copy to
|
|
|
|
* @src_pg: source page
|
|
|
|
* @src_off: offset in page to copy from
|
|
|
|
* @len: length
|
|
|
|
*
|
|
|
|
* Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
|
|
|
|
* address according to the DMA mapping API rules for streaming mappings.
|
|
|
|
* Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
|
|
|
|
* (kernel memory or locked user space pages).
|
|
|
|
*/
|
|
|
|
dma_cookie_t
|
|
|
|
dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
|
|
|
|
unsigned int dest_off, struct page *src_pg, unsigned int src_off,
|
|
|
|
size_t len)
|
|
|
|
{
|
|
|
|
struct dma_device *dev = chan->device;
|
|
|
|
struct dma_async_tx_descriptor *tx;
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_addr_t dma_dest, dma_src;
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
dma_cookie_t cookie;
|
|
|
|
int cpu;
|
|
|
|
|
2008-02-02 19:49:57 -07:00
|
|
|
dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
|
|
|
|
dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len,
|
|
|
|
DMA_FROM_DEVICE);
|
2008-04-17 21:17:26 -06:00
|
|
|
tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
|
|
|
|
DMA_CTRL_ACK);
|
2008-02-02 19:49:57 -07:00
|
|
|
|
|
|
|
if (!tx) {
|
|
|
|
dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE);
|
|
|
|
dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
return -ENOMEM;
|
2008-02-02 19:49:57 -07:00
|
|
|
}
|
dmaengine: refactor dmaengine around dma_async_tx_descriptor
The current dmaengine interface defines mutliple routines per operation,
i.e. dma_async_memcpy_buf_to_buf, dma_async_memcpy_buf_to_page etc. Adding
more operation types (xor, crc, etc) to this model would result in an
unmanageable number of method permutations.
Are we really going to add a set of hooks for each DMA engine
whizbang feature?
- Jeff Garzik
The descriptor creation process is refactored using the new common
dma_async_tx_descriptor structure. Instead of per driver
do_<operation>_<dest>_to_<src> methods, drivers integrate
dma_async_tx_descriptor into their private software descriptor and then
define a 'prep' routine per operation. The prep routine allocates a
descriptor and ensures that the tx_set_src, tx_set_dest, tx_submit routines
are valid. Descriptor creation and submission becomes:
struct dma_device *dev;
struct dma_chan *chan;
struct dma_async_tx_descriptor *tx;
tx = dev->device_prep_dma_<operation>(chan, len, int_flag)
tx->tx_set_src(dma_addr_t, tx, index /* for multi-source ops */)
tx->tx_set_dest(dma_addr_t, tx, index)
tx->tx_submit(tx)
In addition to the refactoring, dma_async_tx_descriptor also lays the
groundwork for definining cross-channel-operation dependencies, and a
callback facility for asynchronous notification of operation completion.
Changelog:
* drop dma mapping methods, suggested by Chris Leech
* fix ioat_dma_dependency_added, also caught by Andrew Morton
* fix dma_sync_wait, change from Andrew Morton
* uninline large functions, change from Andrew Morton
* add tx->callback = NULL to dmaengine calls to interoperate with async_tx
calls
* hookup ioat_tx_submit
* convert channel capabilities to a 'cpumask_t like' bitmap
* removed DMA_TX_ARRAY_INIT, no longer needed
* checkpatch.pl fixes
* make set_src, set_dest, and tx_submit descriptor specific methods
* fixup git-ioat merge
* move group_list and phys to dma_async_tx_descriptor
Cc: Jeff Garzik <jeff@garzik.org>
Cc: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Shannon Nelson <shannon.nelson@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
2007-01-02 11:10:43 -07:00
|
|
|
|
|
|
|
tx->callback = NULL;
|
|
|
|
cookie = tx->tx_submit(tx);
|
|
|
|
|
|
|
|
cpu = get_cpu();
|
|
|
|
per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
|
|
|
|
per_cpu_ptr(chan->local, cpu)->memcpy_count++;
|
|
|
|
put_cpu();
|
|
|
|
|
|
|
|
return cookie;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
|
|
|
|
|
|
|
|
void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
|
|
|
|
struct dma_chan *chan)
|
|
|
|
{
|
|
|
|
tx->chan = chan;
|
|
|
|
spin_lock_init(&tx->lock);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(dma_async_tx_descriptor_init);
|
|
|
|
|
2009-01-05 17:14:31 -07:00
|
|
|
/* dma_wait_for_async_tx - spin wait for a transaction to complete
|
|
|
|
* @tx: in-flight transaction to wait on
|
|
|
|
*
|
|
|
|
* This routine assumes that tx was obtained from a call to async_memcpy,
|
|
|
|
* async_xor, async_memset, etc which ensures that tx is "in-flight" (prepped
|
|
|
|
* and submitted). Walking the parent chain is only meant to cover for DMA
|
|
|
|
* drivers that do not implement the DMA_INTERRUPT capability and may race with
|
|
|
|
* the driver's descriptor cleanup routine.
|
|
|
|
*/
|
|
|
|
enum dma_status
|
|
|
|
dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
|
|
|
|
{
|
|
|
|
enum dma_status status;
|
|
|
|
struct dma_async_tx_descriptor *iter;
|
|
|
|
struct dma_async_tx_descriptor *parent;
|
|
|
|
|
|
|
|
if (!tx)
|
|
|
|
return DMA_SUCCESS;
|
|
|
|
|
|
|
|
WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for"
|
2009-01-06 11:38:21 -07:00
|
|
|
" %s\n", __func__, dma_chan_name(tx->chan));
|
2009-01-05 17:14:31 -07:00
|
|
|
|
|
|
|
/* poll through the dependency chain, return when tx is complete */
|
|
|
|
do {
|
|
|
|
iter = tx;
|
|
|
|
|
|
|
|
/* find the root of the unsubmitted dependency chain */
|
|
|
|
do {
|
|
|
|
parent = iter->parent;
|
|
|
|
if (!parent)
|
|
|
|
break;
|
|
|
|
else
|
|
|
|
iter = parent;
|
|
|
|
} while (parent);
|
|
|
|
|
|
|
|
/* there is a small window for ->parent == NULL and
|
|
|
|
* ->cookie == -EBUSY
|
|
|
|
*/
|
|
|
|
while (iter->cookie == -EBUSY)
|
|
|
|
cpu_relax();
|
|
|
|
|
|
|
|
status = dma_sync_wait(iter->chan, iter->cookie);
|
|
|
|
} while (status == DMA_IN_PROGRESS || (iter != tx));
|
|
|
|
|
|
|
|
return status;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
|
|
|
|
|
|
|
|
/* dma_run_dependencies - helper routine for dma drivers to process
|
|
|
|
* (start) dependent operations on their target channel
|
|
|
|
* @tx: transaction with dependencies
|
|
|
|
*/
|
|
|
|
void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
|
|
|
|
{
|
|
|
|
struct dma_async_tx_descriptor *dep = tx->next;
|
|
|
|
struct dma_async_tx_descriptor *dep_next;
|
|
|
|
struct dma_chan *chan;
|
|
|
|
|
|
|
|
if (!dep)
|
|
|
|
return;
|
|
|
|
|
2009-01-12 15:17:20 -07:00
|
|
|
/* we'll submit tx->next now, so clear the link */
|
|
|
|
tx->next = NULL;
|
2009-01-05 17:14:31 -07:00
|
|
|
chan = dep->chan;
|
|
|
|
|
|
|
|
/* keep submitting up until a channel switch is detected
|
|
|
|
* in that case we will be called again as a result of
|
|
|
|
* processing the interrupt from async_tx_channel_switch
|
|
|
|
*/
|
|
|
|
for (; dep; dep = dep_next) {
|
|
|
|
spin_lock_bh(&dep->lock);
|
|
|
|
dep->parent = NULL;
|
|
|
|
dep_next = dep->next;
|
|
|
|
if (dep_next && dep_next->chan == chan)
|
|
|
|
dep->next = NULL; /* ->next will be submitted */
|
|
|
|
else
|
|
|
|
dep_next = NULL; /* submit current dep and terminate */
|
|
|
|
spin_unlock_bh(&dep->lock);
|
|
|
|
|
|
|
|
dep->tx_submit(dep);
|
|
|
|
}
|
|
|
|
|
|
|
|
chan->device->device_issue_pending(chan);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(dma_run_dependencies);
|
|
|
|
|
2006-05-23 18:18:44 -06:00
|
|
|
static int __init dma_bus_init(void)
|
|
|
|
{
|
2009-01-06 11:38:21 -07:00
|
|
|
idr_init(&dma_idr);
|
2006-05-23 18:18:44 -06:00
|
|
|
mutex_init(&dma_list_mutex);
|
|
|
|
return class_register(&dma_devclass);
|
|
|
|
}
|
2009-01-06 11:38:22 -07:00
|
|
|
arch_initcall(dma_bus_init);
|
2006-05-23 18:18:44 -06:00
|
|
|
|
2009-01-06 11:38:14 -07:00
|
|
|
|