6afd227ca1
By design, workgroups are queued in the form of linked lists. Previously, it points to the next `z_erofs_vle_workgroup', which isn't flexible enough to simplify `z_erofs_vle_submit_all'. Let's fix it by pointing to the next `owned_workgrp_t' and use container_of to get its coresponding `z_erofs_vle_workgroup'. Reviewed-by: Chao Yu <yuchao0@huawei.com> Signed-off-by: Gao Xiang <gaoxiang25@huawei.com> Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
230 lines
5.7 KiB
C
230 lines
5.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0
|
|
*
|
|
* linux/drivers/staging/erofs/unzip_vle.h
|
|
*
|
|
* Copyright (C) 2018 HUAWEI, Inc.
|
|
* http://www.huawei.com/
|
|
* Created by Gao Xiang <gaoxiang25@huawei.com>
|
|
*
|
|
* This file is subject to the terms and conditions of the GNU General Public
|
|
* License. See the file COPYING in the main directory of the Linux
|
|
* distribution for more details.
|
|
*/
|
|
#ifndef __EROFS_FS_UNZIP_VLE_H
|
|
#define __EROFS_FS_UNZIP_VLE_H
|
|
|
|
#include "internal.h"
|
|
#include "unzip_pagevec.h"
|
|
|
|
/*
|
|
* - 0x5A110C8D ('sallocated', Z_EROFS_MAPPING_STAGING) -
|
|
* used for temporary allocated pages (via erofs_allocpage),
|
|
* in order to seperate those from NULL mapping (eg. truncated pages)
|
|
*/
|
|
#define Z_EROFS_MAPPING_STAGING ((void *)0x5A110C8D)
|
|
|
|
#define z_erofs_is_stagingpage(page) \
|
|
((page)->mapping == Z_EROFS_MAPPING_STAGING)
|
|
|
|
static inline bool z_erofs_gather_if_stagingpage(struct list_head *page_pool,
|
|
struct page *page)
|
|
{
|
|
if (z_erofs_is_stagingpage(page)) {
|
|
list_add(&page->lru, page_pool);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* Structure fields follow one of the following exclusion rules.
|
|
*
|
|
* I: Modifiable by initialization/destruction paths and read-only
|
|
* for everyone else.
|
|
*
|
|
*/
|
|
|
|
#define Z_EROFS_VLE_INLINE_PAGEVECS 3
|
|
|
|
struct z_erofs_vle_work {
|
|
struct mutex lock;
|
|
|
|
/* I: decompression offset in page */
|
|
unsigned short pageofs;
|
|
unsigned short nr_pages;
|
|
|
|
/* L: queued pages in pagevec[] */
|
|
unsigned vcnt;
|
|
|
|
union {
|
|
/* L: pagevec */
|
|
erofs_vtptr_t pagevec[Z_EROFS_VLE_INLINE_PAGEVECS];
|
|
struct rcu_head rcu;
|
|
};
|
|
};
|
|
|
|
#define Z_EROFS_VLE_WORKGRP_FMT_PLAIN 0
|
|
#define Z_EROFS_VLE_WORKGRP_FMT_LZ4 1
|
|
#define Z_EROFS_VLE_WORKGRP_FMT_MASK 1
|
|
|
|
typedef void *z_erofs_vle_owned_workgrp_t;
|
|
|
|
struct z_erofs_vle_workgroup {
|
|
struct erofs_workgroup obj;
|
|
struct z_erofs_vle_work work;
|
|
|
|
/* point to next owned_workgrp_t */
|
|
z_erofs_vle_owned_workgrp_t next;
|
|
|
|
/* compressed pages (including multi-usage pages) */
|
|
struct page *compressed_pages[Z_EROFS_CLUSTER_MAX_PAGES];
|
|
unsigned int llen, flags;
|
|
};
|
|
|
|
/* let's avoid the valid 32-bit kernel addresses */
|
|
|
|
/* the chained workgroup has't submitted io (still open) */
|
|
#define Z_EROFS_VLE_WORKGRP_TAIL ((void *)0x5F0ECAFE)
|
|
/* the chained workgroup has already submitted io */
|
|
#define Z_EROFS_VLE_WORKGRP_TAIL_CLOSED ((void *)0x5F0EDEAD)
|
|
|
|
#define Z_EROFS_VLE_WORKGRP_NIL (NULL)
|
|
|
|
#define z_erofs_vle_workgrp_fmt(grp) \
|
|
((grp)->flags & Z_EROFS_VLE_WORKGRP_FMT_MASK)
|
|
|
|
static inline void z_erofs_vle_set_workgrp_fmt(
|
|
struct z_erofs_vle_workgroup *grp,
|
|
unsigned int fmt)
|
|
{
|
|
grp->flags = fmt | (grp->flags & ~Z_EROFS_VLE_WORKGRP_FMT_MASK);
|
|
}
|
|
|
|
|
|
/* definitions if multiref is disabled */
|
|
#define z_erofs_vle_grab_primary_work(grp) (&(grp)->work)
|
|
#define z_erofs_vle_grab_work(grp, pageofs) (&(grp)->work)
|
|
#define z_erofs_vle_work_workgroup(wrk, primary) \
|
|
((primary) ? container_of(wrk, \
|
|
struct z_erofs_vle_workgroup, work) : \
|
|
({ BUG(); (void *)NULL; }))
|
|
|
|
|
|
#define Z_EROFS_WORKGROUP_SIZE sizeof(struct z_erofs_vle_workgroup)
|
|
|
|
struct z_erofs_vle_unzip_io {
|
|
atomic_t pending_bios;
|
|
z_erofs_vle_owned_workgrp_t head;
|
|
|
|
union {
|
|
wait_queue_head_t wait;
|
|
struct work_struct work;
|
|
} u;
|
|
};
|
|
|
|
struct z_erofs_vle_unzip_io_sb {
|
|
struct z_erofs_vle_unzip_io io;
|
|
struct super_block *sb;
|
|
};
|
|
|
|
#define Z_EROFS_ONLINEPAGE_COUNT_BITS 2
|
|
#define Z_EROFS_ONLINEPAGE_COUNT_MASK ((1 << Z_EROFS_ONLINEPAGE_COUNT_BITS) - 1)
|
|
#define Z_EROFS_ONLINEPAGE_INDEX_SHIFT (Z_EROFS_ONLINEPAGE_COUNT_BITS)
|
|
|
|
/*
|
|
* waiters (aka. ongoing_packs): # to unlock the page
|
|
* sub-index: 0 - for partial page, >= 1 full page sub-index
|
|
*/
|
|
typedef atomic_t z_erofs_onlinepage_t;
|
|
|
|
/* type punning */
|
|
union z_erofs_onlinepage_converter {
|
|
z_erofs_onlinepage_t *o;
|
|
unsigned long *v;
|
|
};
|
|
|
|
static inline unsigned z_erofs_onlinepage_index(struct page *page)
|
|
{
|
|
union z_erofs_onlinepage_converter u;
|
|
|
|
BUG_ON(!PagePrivate(page));
|
|
u.v = &page_private(page);
|
|
|
|
return atomic_read(u.o) >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
|
|
}
|
|
|
|
static inline void z_erofs_onlinepage_init(struct page *page)
|
|
{
|
|
union {
|
|
z_erofs_onlinepage_t o;
|
|
unsigned long v;
|
|
/* keep from being unlocked in advance */
|
|
} u = { .o = ATOMIC_INIT(1) };
|
|
|
|
set_page_private(page, u.v);
|
|
smp_wmb();
|
|
SetPagePrivate(page);
|
|
}
|
|
|
|
static inline void z_erofs_onlinepage_fixup(struct page *page,
|
|
uintptr_t index, bool down)
|
|
{
|
|
unsigned long *p, o, v, id;
|
|
repeat:
|
|
p = &page_private(page);
|
|
o = READ_ONCE(*p);
|
|
|
|
id = o >> Z_EROFS_ONLINEPAGE_INDEX_SHIFT;
|
|
if (id) {
|
|
if (!index)
|
|
return;
|
|
|
|
BUG_ON(id != index);
|
|
}
|
|
|
|
v = (index << Z_EROFS_ONLINEPAGE_INDEX_SHIFT) |
|
|
((o & Z_EROFS_ONLINEPAGE_COUNT_MASK) + (unsigned)down);
|
|
if (cmpxchg(p, o, v) != o)
|
|
goto repeat;
|
|
}
|
|
|
|
static inline void z_erofs_onlinepage_endio(struct page *page)
|
|
{
|
|
union z_erofs_onlinepage_converter u;
|
|
unsigned v;
|
|
|
|
BUG_ON(!PagePrivate(page));
|
|
u.v = &page_private(page);
|
|
|
|
v = atomic_dec_return(u.o);
|
|
if (!(v & Z_EROFS_ONLINEPAGE_COUNT_MASK)) {
|
|
ClearPagePrivate(page);
|
|
if (!PageError(page))
|
|
SetPageUptodate(page);
|
|
unlock_page(page);
|
|
}
|
|
|
|
debugln("%s, page %p value %x", __func__, page, atomic_read(u.o));
|
|
}
|
|
|
|
#define Z_EROFS_VLE_VMAP_ONSTACK_PAGES \
|
|
min_t(unsigned int, THREAD_SIZE / 8 / sizeof(struct page *), 96U)
|
|
#define Z_EROFS_VLE_VMAP_GLOBAL_PAGES 2048
|
|
|
|
/* unzip_vle_lz4.c */
|
|
extern int z_erofs_vle_plain_copy(struct page **compressed_pages,
|
|
unsigned clusterpages, struct page **pages,
|
|
unsigned nr_pages, unsigned short pageofs);
|
|
|
|
extern int z_erofs_vle_unzip_fast_percpu(struct page **compressed_pages,
|
|
unsigned clusterpages, struct page **pages,
|
|
unsigned outlen, unsigned short pageofs,
|
|
void (*endio)(struct page *));
|
|
|
|
extern int z_erofs_vle_unzip_vmap(struct page **compressed_pages,
|
|
unsigned clusterpages, void *vaddr, unsigned llen,
|
|
unsigned short pageofs, bool overlapped);
|
|
|
|
#endif
|
|
|