diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 90117fb13ec8e0..a5127324709bab 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -370,24 +370,27 @@ void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) } } +/* + * fuse_args_to_req() assigns the unique already, so that the early tracepoints + * see it. Assign here only for requests that did not pass through it. + * + * The send is not traced here: this runs where a request is queued, which + * trace_fuse_request_enqueue() already marks. trace_fuse_request_send() + * belongs where the request reaches the server, in fuse_dev_do_read() and + * fuse_uring_send(). + */ static inline void fuse_request_assign_unique_locked(struct fuse_iqueue *fiq, struct fuse_req *req) { - if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + if (!req->in.h.unique && req->in.h.opcode != FUSE_NOTIFY_REPLY) req->in.h.unique = fuse_get_unique_locked(fiq); - - /* tracepoint captures in.h.unique and in.h.len */ - trace_fuse_request_send(req); } inline void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req) { - if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + if (!req->in.h.unique && req->in.h.opcode != FUSE_NOTIFY_REPLY) req->in.h.unique = fuse_get_unique(fiq); - - /* tracepoint captures in.h.unique and in.h.len */ - trace_fuse_request_send(req); } EXPORT_SYMBOL_GPL(fuse_request_assign_unique); @@ -662,6 +665,9 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; if (args->end) __set_bit(FR_ASYNC, &req->flags); + + if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + req->in.h.unique = fuse_get_unique(&req->fm->fc->iq); } ssize_t fuse_compound_request(struct fuse_mount *fm, struct fuse_args *args) diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 7fa79e68afd5c9..88662d909fe8bd 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -12,6 +12,7 @@ #include #include #include +#include static bool __read_mostly enable_uring; module_param(enable_uring, bool, 0644); @@ -28,7 +29,6 @@ MODULE_PARM_DESC(enable_uring, /* Number of (re)tries to find a better queue */ #define FUSE_URING_Q_TRIES 3 - bool fuse_uring_enabled(void) { return enable_uring; @@ -85,8 +85,8 @@ static void fuse_uring_flush_queue_bg(struct fuse_ring_queue *queue) } } -static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, - int error) +static void __fuse_uring_req_end(struct fuse_ring_ent *ent, + struct fuse_req *req, int error) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; @@ -109,6 +109,43 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, req->out.h.error = error; clear_bit(FR_SENT, &req->flags); +} + +static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, + int error) +{ + __fuse_uring_req_end(ent, req, error); + fuse_request_end(req); +} + +static void fuse_uring_req_end_work(struct callback_head *work) +{ + struct fuse_req *req = container_of(work, struct fuse_req, + ring_end_work); + + fuse_request_end(req); +} + +/* + * On the commit path ->uring_cmd() runs with ctx->uring_lock held by + * io_uring_enter(). fuse_request_end() wakes the request submitter, which + * typically preempts the ring task right away (same CPU) - while the mutex + * is still held. Defer the completion to task work, which runs once the + * submission path has released the lock (in io_cqring_wait() or on return + * to userspace), so the ring task can finish its critical section first. + */ +static void fuse_uring_req_end_deferred(struct fuse_ring_ent *ent, + struct fuse_req *req, int error, + unsigned int issue_flags) +{ + __fuse_uring_req_end(ent, req, error); + + if (!(issue_flags & IO_URING_F_UNLOCKED)) { + init_task_work(&req->ring_end_work, fuse_uring_req_end_work); + if (!task_work_add(current, &req->ring_end_work, TWA_RESUME)) + return; + } + fuse_request_end(req); } @@ -153,41 +190,6 @@ void fuse_uring_flush_bg(struct fuse_conn *fc) } } -/* - * Copy from memmap.c, should be exported - */ -static void io_pages_free(struct page ***pages, int npages) -{ - struct page **page_array = *pages; - - if (!page_array) - return; - - unpin_user_pages(page_array, npages); - kvfree(page_array); - *pages = NULL; -} - - -static void fuse_ring_destruct_q_map(struct fuse_queue_map *q_map) -{ - free_cpumask_var(q_map->registered_q_mask); - kfree(q_map->cpu_to_qid); -} - -static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) -{ - int node; - - fuse_ring_destruct_q_map(&ring->q_map); - - if (ring->numa_q_map) { - for (node = 0; node < ring->nr_numa_nodes; node++) - fuse_ring_destruct_q_map(&ring->numa_q_map[node]); - kfree(ring->numa_q_map); - } -} - static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list) { struct fuse_ring_ent *ent; @@ -231,6 +233,40 @@ bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } +/* + * Copy from memmap.c, should be exported + */ +static void io_pages_free(struct page ***pages, int npages) +{ + struct page **page_array = *pages; + + if (!page_array) + return; + + unpin_user_pages(page_array, npages); + kvfree(page_array); + *pages = NULL; +} + +static void fuse_ring_destruct_q_map(struct fuse_queue_map *q_map) +{ + free_cpumask_var(q_map->registered_q_mask); + kfree(q_map->cpu_to_qid); +} + +static void fuse_uring_destruct_q_masks(struct fuse_ring *ring) +{ + int node; + + fuse_ring_destruct_q_map(&ring->q_map); + + if (ring->numa_q_map) { + for (node = 0; node < ring->nr_numa_nodes; node++) + fuse_ring_destruct_q_map(&ring->numa_q_map[node]); + kfree(ring->numa_q_map); + } +} + void fuse_uring_destruct(struct fuse_conn *fc) { struct fuse_ring *ring = fc->ring; @@ -323,8 +359,8 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc) ring->nr_numa_nodes = num_online_nodes(); - ring->queues = kcalloc(nr_queues, sizeof(struct fuse_ring_queue *), - GFP_KERNEL_ACCOUNT); + ring->queues = kzalloc_objs(struct fuse_ring_queue *, nr_queues, + GFP_KERNEL_ACCOUNT); if (!ring->queues) goto out_err; @@ -849,7 +885,6 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, /* copy the payload */ err = fuse_copy_args(&cs, num_args, args->in_pages, (struct fuse_arg *)in_args, 0); - fuse_copy_finish(&cs); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); goto copy_finish; @@ -921,6 +956,21 @@ static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, return err; } +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move_tail(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + spin_unlock(&queue->lock); + + trace_fuse_request_send(ent->fuse_req); + io_uring_cmd_done(cmd, ret, issue_flags); +} + /* * Write data to the ring buffer and send the request to userspace, * userspace will read it @@ -930,22 +980,13 @@ static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent, struct fuse_req *req, unsigned int issue_flags) { - struct fuse_ring_queue *queue = ent->queue; int err; - struct io_uring_cmd *cmd; err = fuse_uring_prepare_send(ent, req); if (err) return err; - spin_lock(&queue->lock); - cmd = ent->cmd; - ent->cmd = NULL; - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - spin_unlock(&queue->lock); - - io_uring_cmd_done(cmd, 0, issue_flags); + fuse_uring_send(ent, ent->cmd, 0, issue_flags); return 0; } @@ -1041,7 +1082,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, err = fuse_uring_copy_from_ring(ring, req, ent); out: - fuse_uring_req_end(ent, req, err); + fuse_uring_req_end_deferred(ent, req, err, issue_flags); } /* @@ -1459,21 +1500,6 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) return -EIOCBQUEUED; } -static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, - ssize_t ret, unsigned int issue_flags) -{ - struct fuse_ring_queue *queue = ent->queue; - - spin_lock(&queue->lock); - ent->state = FRRS_USERSPACE; - list_move_tail(&ent->list, &queue->ent_in_userspace); - ent->cmd = NULL; - spin_unlock(&queue->lock); - - trace_fuse_request_send(ent->fuse_req); - io_uring_cmd_done(cmd, ret, issue_flags); -} - /* * This prepares and sends the ring request in fuse-uring task context. * User buffers are not mapped yet - the application does not have permission @@ -1500,6 +1526,28 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) fuse_uring_send(ent, cmd, err, issue_flags); } +/* + * The request was already copied to the ring buffer in the submitter's + * context, only the io_uring cmd completion is left to do. + * io_uring_cmd_done() must not run in the submitter's context as it would + * have to take ctx->uring_lock (io_uring_cmd_del_cancelable()) - a mutex + * the ring task holds across its whole submission path and frequently gets + * preempted under while the just-woken submitter runs. + */ +static void fuse_uring_send_prepared_in_task(struct io_tw_req tw_req, + io_tw_token_t tw) +{ + unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + int err = 0; + + if (unlikely(tw.cancel)) + err = -ECANCELED; + + fuse_uring_send(ent, cmd, err, issue_flags); +} + static struct fuse_ring_queue *fuse_uring_select_queue(struct fuse_ring *ring, bool background) { @@ -1599,7 +1647,9 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent, bool bg) IO_URING_F_UNLOCKED); return; } - fuse_uring_send(ent, cmd, 0, IO_URING_F_UNLOCKED); + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, + fuse_uring_send_prepared_in_task); } } diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h index 4518990e98bdd5..667f668ef3277f 100644 --- a/fs/fuse/dev_uring_i.h +++ b/fs/fuse/dev_uring_i.h @@ -231,10 +231,6 @@ static inline bool fuse_uring_request_expired(struct fuse_conn *fc) return false; } -static inline bool fuse_uring_request_expired(struct fuse_conn *fc) -{ -} - static inline void fuse_uring_flush_bg(struct fuse_conn *fc) { } diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 05c081ca0ca59d..b19ec2669f9d03 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -28,6 +28,8 @@ module_param(allow_sys_admin_access, bool, 0644); MODULE_PARM_DESC(allow_sys_admin_access, "Allow users with CAP_SYS_ADMIN in initial userns to bypass allow_other access check"); +static void fuse_attr_to_statx(struct fuse_attr *attr, struct fuse_statx *sx, uint32_t mask); + struct dentry_bucket { struct rb_root tree; spinlock_t lock; @@ -373,6 +375,73 @@ static void fuse_lookup_init(struct fuse_conn *fc, struct fuse_args *args, args->out_args[0].value = outarg; } +/** + * fuse_do_lookupx - Perform a FUSE_LOOKUPX operation + * + * @ext_out: extended output argument structure + * @lookup_flags: lookup flags (e.g., FUSE_LOOKUPX_FOR_REVALIDATE) + */ +static int fuse_do_lookupx(struct fuse_mount *fm, u64 nodeid, + const struct qstr *name, + struct fuse_lookupx_out *ext_out, + uint32_t lookup_flags) +{ + struct fuse_conn *fc = fm->fc; + FUSE_ARGS(args); + struct fuse_lookupx_in inarg = { .lookup_flags = lookup_flags }; + int err; + + memset(ext_out, 0, sizeof(*ext_out)); + args.nodeid = nodeid; + + if (!fc->lookupx) + goto fallback; + + args.opcode = FUSE_LOOKUPX; + args.in_numargs = 4; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = 0; + args.in_args[1].value = NULL; + args.in_args[2].size = name->len; + args.in_args[2].value = name->name; + args.in_args[3].size = 1; + args.in_args[3].value = ""; + args.out_numargs = 1; + args.out_args[0].size = sizeof(struct fuse_lookupx_out); + args.out_args[0].value = ext_out; + + err = fuse_simple_request(fm, &args); + if (err) { + if (err == -ENOSYS) { + fc->lookupx = 0; + goto fallback; + } + return err; + } + + return 0; + +fallback: + args.opcode = FUSE_LOOKUP; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = name->len; + args.in_args[1].value = name->name; + args.in_args[2].size = 1; + args.in_args[2].value = ""; + args.out_numargs = 1; + args.out_args[0].size = sizeof(struct fuse_entry_out); + args.out_args[0].value = &ext_out->entry; + + err = fuse_simple_request(fm, &args); + if (err) + return err; + + ext_out->mask = STATX_BASIC_STATS; + return 0; +} + /* * Check whether the dentry is still valid * @@ -400,10 +469,11 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, goto invalid; else if (time_before64(fuse_dentry_time(entry), get_jiffies_64()) || (flags & (LOOKUP_EXCL | LOOKUP_REVAL | LOOKUP_RENAME_TARGET))) { - struct fuse_entry_out outarg; - FUSE_ARGS(args); + struct fuse_lookupx_out ext_out; + struct fuse_statx sx; struct fuse_forget_link *forget; u64 attr_version; + uint32_t lookupx_flags = FUSE_LOOKUPX_FOR_REVALIDATE; /* For negative dentries, always do a fresh lookup */ if (!inode) @@ -421,19 +491,21 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, goto out; attr_version = fuse_get_attr_version(fm->fc); + if (S_ISDIR(inode->i_mode)) + lookupx_flags |= FUSE_LOOKUPX_TARGET_WAS_DIR; - fuse_lookup_init(fm->fc, &args, get_node_id(dir), - name, &outarg); - ret = fuse_simple_request(fm, &args); + ret = fuse_do_lookupx(fm, get_node_id(dir), + name, &ext_out, + lookupx_flags); /* Zero nodeid is same as -ENOENT */ - if (!ret && !outarg.nodeid) + if (!ret && !ext_out.entry.nodeid) ret = -ENOENT; if (!ret) { fi = get_fuse_inode(inode); - if (outarg.nodeid != get_node_id(inode) || - (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) { + if (ext_out.entry.nodeid != get_node_id(inode) || + (bool) IS_AUTOMOUNT(inode) != (bool) (ext_out.entry.attr.flags & FUSE_ATTR_SUBMOUNT)) { fuse_queue_forget(fm->fc, forget, - outarg.nodeid, 1); + ext_out.entry.nodeid, 1); goto invalid; } spin_lock(&fi->lock); @@ -443,15 +515,16 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, kfree(forget); if (ret == -ENOMEM || ret == -EINTR) goto out; - if (ret || fuse_invalid_attr(&outarg.attr) || - fuse_stale_inode(inode, outarg.generation, &outarg.attr)) + if (ret || fuse_invalid_attr(&ext_out.entry.attr) || + fuse_stale_inode(inode, ext_out.entry.generation, &ext_out.entry.attr)) goto invalid; forget_all_cached_acls(inode); - fuse_change_attributes(inode, &outarg.attr, NULL, - ATTR_TIMEOUT(&outarg), + fuse_attr_to_statx(&ext_out.entry.attr, &sx, ext_out.mask); + fuse_change_attributes(inode, &ext_out.entry.attr, &sx, + ATTR_TIMEOUT(&ext_out.entry), attr_version); - fuse_change_entry_timeout(entry, &outarg); + fuse_change_entry_timeout(entry, &ext_out.entry); } else if (inode) { fi = get_fuse_inode(inode); if (flags & LOOKUP_RCU) { @@ -846,6 +919,10 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, memset(&inarg, 0, sizeof(inarg)); memset(&outentry, 0, sizeof(outentry)); inarg.flags = flags; + + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; inarg.mode = mode; inarg.umask = current_umask(); @@ -1411,8 +1488,35 @@ static void fuse_statx_to_attr(struct fuse_statx *sx, struct fuse_attr *attr) attr->blksize = sx->blksize; } +static void fuse_attr_to_statx(struct fuse_attr *attr, struct fuse_statx *sx, uint32_t mask) +{ + memset(sx, 0, sizeof(*sx)); + sx->mask = mask; + sx->ino = attr->ino; + sx->size = attr->size; + sx->blocks = attr->blocks; + sx->atime.tv_sec = attr->atime; + sx->mtime.tv_sec = attr->mtime; + sx->ctime.tv_sec = attr->ctime; + sx->atime.tv_nsec = attr->atimensec; + sx->mtime.tv_nsec = attr->mtimensec; + sx->ctime.tv_nsec = attr->ctimensec; + sx->mode = attr->mode; + sx->nlink = attr->nlink; + sx->uid = attr->uid; + sx->gid = attr->gid; + sx->rdev_major = MAJOR(attr->rdev); + sx->rdev_minor = MINOR(attr->rdev); + sx->blksize = attr->blksize; +} + +/* + * @param sx_mask request mask send to to fuse-server + * @param mandatory_sx_mask subset of (or complete) sx_mask that the server + * has to fulfill +*/ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, - struct file *file, struct kstat *stat) + struct file *file, struct kstat *stat, u32 sx_mask, u32 mandatory_sx_mask) { int err; struct fuse_attr attr; @@ -1423,6 +1527,12 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, u64 attr_version = fuse_get_attr_version(fm->fc); FUSE_ARGS(args); + /* + * mandatory_sx_mask should be a subset of sx_mask. + * If it's not, we have a logic error somewhere in the call chain. + */ + WARN_ON_ONCE((mandatory_sx_mask & sx_mask) != mandatory_sx_mask); + memset(&inarg, 0, sizeof(inarg)); memset(&outarg, 0, sizeof(outarg)); /* Directories have separate file-handle space */ @@ -1432,9 +1542,12 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, inarg.getattr_flags |= FUSE_GETATTR_FH; inarg.fh = ff->fh; } - /* For now leave sync hints as the default, request all stats. */ + /* + * For permission checks, we only need mode, uid, gid. + * This is an optimization to avoid fetching all stats when not needed. + */ inarg.sx_flags = 0; - inarg.sx_mask = STATX_BASIC_STATS | STATX_BTIME; + inarg.sx_mask = sx_mask; args.opcode = FUSE_STATX; args.nodeid = get_node_id(inode); args.in_numargs = 1; @@ -1448,6 +1561,17 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, return err; sx = &outarg.stat; + + /* + * Verify the server returned at least what we requested. + * The server may return more attributes than requested (which is fine), + * but must not return fewer. + */ + if ((sx->mask & mandatory_sx_mask) != mandatory_sx_mask) { + fuse_make_bad(inode); + return -EIO; + } + if (((sx->mask & STATX_SIZE) && !fuse_valid_size(sx->size)) || ((sx->mask & STATX_TYPE) && (!fuse_valid_type(sx->mode) || inode_wrong_type(inode, sx->mode)))) { @@ -1456,7 +1580,7 @@ static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, } fuse_statx_to_attr(&outarg.stat, &attr); - if ((sx->mask & STATX_BASIC_STATS) == STATX_BASIC_STATS) { + if (sx->mask & STATX_BASIC_STATS) { fuse_change_attributes(inode, &attr, &outarg.stat, ATTR_TIMEOUT(&outarg), attr_version); } @@ -1521,30 +1645,31 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, bool sync; u32 inval_mask = READ_ONCE(fi->inval_mask); u32 cache_mask = fuse_get_cache_mask(inode); - + u32 mandatory_sx_mask = request_mask & STATX_BASIC_STATS; + u32 sx_mask = request_mask; /* FUSE only supports basic stats and possibly btime */ - request_mask &= STATX_BASIC_STATS | STATX_BTIME; + sx_mask &= STATX_BASIC_STATS | STATX_BTIME; retry: if (fc->no_statx) - request_mask &= STATX_BASIC_STATS; + sx_mask &= STATX_BASIC_STATS; - if (!request_mask) + if (!sx_mask) sync = false; else if (flags & AT_STATX_FORCE_SYNC) sync = true; else if (flags & AT_STATX_DONT_SYNC) sync = false; - else if (request_mask & inval_mask & ~cache_mask) + else if (sx_mask & inval_mask & ~cache_mask) sync = true; else sync = time_before64(fi->i_time, get_jiffies_64()); if (sync) { forget_all_cached_acls(inode); - /* Try statx if BTIME is requested */ - if (!fc->no_statx && (request_mask & ~STATX_BASIC_STATS)) { - err = fuse_do_statx(idmap, inode, file, stat); + if (!fc->no_statx) { + err = fuse_do_statx(idmap, inode, file, stat, sx_mask, + mandatory_sx_mask); if (err == -ENOSYS) { fc->no_statx = 1; err = 0; @@ -1554,7 +1679,7 @@ static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, err = fuse_do_getattr(idmap, inode, stat, file); } } else if (stat) { - generic_fillattr(idmap, request_mask, inode, stat); + generic_fillattr(idmap, sx_mask, inode, stat); stat->mode = fi->orig_i_mode; stat->ino = fi->orig_ino; stat->blksize = 1 << fi->cached_i_blkbits; @@ -1721,13 +1846,14 @@ static int fuse_access(struct inode *inode, int mask) return err; } -static int fuse_perm_getattr(struct inode *inode, int mask) +static int fuse_perm_getattr(struct inode *inode, int mask, int perm_mask) { if (mask & MAY_NOT_BLOCK) return -ECHILD; forget_all_cached_acls(inode); - return fuse_do_getattr(&nop_mnt_idmap, inode, NULL, NULL); + return fuse_update_get_attr(&nop_mnt_idmap, inode, NULL, NULL, perm_mask, + AT_STATX_FORCE_SYNC); } /* @@ -1749,6 +1875,7 @@ static int fuse_permission(struct mnt_idmap *idmap, struct fuse_conn *fc = get_fuse_conn(inode); bool refreshed = false; int err = 0; + int perm_mask = STATX_MODE | STATX_UID | STATX_GID; if (fuse_is_bad(inode)) return -EIO; @@ -1762,13 +1889,12 @@ static int fuse_permission(struct mnt_idmap *idmap, if (fc->default_permissions || ((mask & MAY_EXEC) && S_ISREG(inode->i_mode))) { struct fuse_inode *fi = get_fuse_inode(inode); - u32 perm_mask = STATX_MODE | STATX_UID | STATX_GID; if (perm_mask & READ_ONCE(fi->inval_mask) || - time_before64(fi->i_time, get_jiffies_64())) { + time_before64(fi->i_perm_time, get_jiffies_64())) { refreshed = true; - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (err) return err; } @@ -1781,7 +1907,7 @@ static int fuse_permission(struct mnt_idmap *idmap, attributes. This is also needed, because the root node will at first have no permissions */ if (err == -EACCES && !refreshed) { - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (!err) err = generic_permission(idmap, inode, mask); @@ -1798,7 +1924,7 @@ static int fuse_permission(struct mnt_idmap *idmap, if (refreshed) return -EACCES; - err = fuse_perm_getattr(inode, mask); + err = fuse_perm_getattr(inode, mask, perm_mask); if (!err && !(inode->i_mode & S_IXUGO)) return -EACCES; } @@ -2176,35 +2302,24 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. * - * Revoke and drop under the coherency gate write side, - * like the NOTIFY invalidate path: a gate reader that - * already re-validated its grant must not have the - * lock tree and the cache yanked mid-hold, or it - * would repopulate the truncated range trusting a - * grant that no longer exists. Waiting for gate - * readers here is safe: we hold i_rwsem exclusive, so - * no gate holder can be waiting on it (the write path - * takes i_rwsem before the gate, the read path never - * takes it). + * Dropping every grant here does not need a reader or + * writer fenced out: truncate_pagecache() discards the + * folios rather than writing them, and a write racing + * this is a write racing an O_TRUNC open, which has no + * order to preserve. i_rwsem is held exclusive + * anyway, so no cached write is in progress. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); - fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); - if (wb_sem) - percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2295,13 +2410,6 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, /* see the comment in fuse_change_attributes() */ if (!is_wb || is_truncate) i_size_write(inode, outarg.attr.size); - /* - * A truncate settles the size on the server; only shrink the - * server-materialized bound: growing just exposes zeros, which the - * bound need not cover (see fuse_iomap_read_folio_range()). - */ - if (is_truncate && (loff_t) outarg.attr.size < fi->server_size) - fi->server_size = outarg.attr.size; if (is_truncate) { /* NOTE: this may release/reacquire fi->lock */ @@ -2315,23 +2423,17 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* - * Revoke and drop under the coherency gate write side; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out gate - * readers cannot deadlock. + * Revoke past the new size and drop what is beyond it; see + * the atomic-O_TRUNC branch above for why this needs nothing + * fenced out. i_rwsem is held exclusive here as well. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, + U64_MAX); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - if (wb_sem) - percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f019dccaeff6d2..25273163fa00cc 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -71,6 +71,17 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; + /* + * With the writeback cache the kernel owns append positioning: + * writeback sends FUSE_WRITE with explicit offsets, and a server + * that opens its backing file O_APPEND has pwrite(2) ignore them + * (Linux appends regardless of offset). Any re-sent or reordered + * run is then placed at EOF: duplicated data and a growing file. + * Do not hand the flag to the server at all. + */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -176,6 +187,10 @@ static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) open_in.flags &= ~O_TRUNC; + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + open_in.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -251,19 +266,19 @@ struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, &attr_outarg, outargp); if (err == -ENOSYS) fc->compound_open_getattr = 0; - if (!err) + if (!err) fuse_change_attributes(inode, &attr_outarg.attr, - NULL, - ATTR_TIMEOUT(&attr_outarg), - fuse_get_attr_version(fc)); - } - if (err == -ENOSYS) { + NULL, + ATTR_TIMEOUT(&attr_outarg), + fuse_get_attr_version(fc)); + } + if (err == -ENOSYS) { err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); - if (!err) { + if (!err) { ff->fh = outargp->fh; ff->open_flags = outargp->open_flags; - } - } + } + } if (err) { if (err != -ENOSYS) { @@ -346,7 +361,6 @@ static void fuse_truncate_update_attr(struct inode *inode, struct file *file) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); - fi->server_size = 0; i_size_write(inode, 0); spin_unlock(&fi->lock); file_update_time(file); @@ -397,10 +411,26 @@ static int fuse_open(struct inode *inode, struct file *file) if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); if (!err) { - if (is_truncate) + if (is_truncate) { + /* + * Every grant goes with the cache, as on the + * fuse_do_setattr() O_TRUNC path: a record left + * behind would keep naming bytes the folios no + * longer hold. i_rwsem is held exclusive + * (is_wb_truncate), so no cached write is mid-record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { + /* + * Write back first: the drop launders whatever it + * finds dirty a folio at a time, a FUSE_WRITE per + * page, where writeback batches the same bytes. + */ + filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); + } } if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); @@ -482,8 +512,8 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * If this release dropped the last writer, fuse_prepare_release() * cleared the forced-direct-IO latch (under fi->lock). Drop any clean * folios a read racing the latch may have repopulated so they cannot be - * served stale once caching mode resumes. No inode lock or - * wb_inval_rwsem: release may run on the fuse server thread (async fput + * served stale once caching mode resumes. No inode lock: release may + * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could * stall the connection. Writes were routed direct while latched, so * only clean folios exist and this invalidate is server-free; the last @@ -674,6 +704,16 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, if (fuse_is_bad(inode)) return -EIO; + /* + * Get the sending out of the way before the lock. It is the long + * part: a pass takes back every grant it finds gone, a cluster round + * trip each, and cached writers hold i_rwsem shared + * (fuse_cache_wr_exclusive_lock()) and would all wait behind it. + * Errors are left to the pass below, which collects them from the + * mapping. + */ + filemap_fdatawrite_range(file->f_mapping, start, end); + inode_lock(inode); /* @@ -1020,6 +1060,107 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, return 0; } +/** + * fuse_read_folio_range - read part of a folio from the server + * @file: file to read through + * @folio: the folio to fill + * @off: offset within @folio to start at + * @len: bytes to read + * + * fuse_do_readfolio() cannot serve a partial folio: it asks for + * page_zeroing, and fuse_copy_folio() answers that by zeroing the whole + * folio whenever the request covers less than all of it. Ask without it + * and zero exactly what the reply left short, which is the server saying + * the file ends there. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_range(struct file *file, struct folio *folio, + size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = folio_pos(folio) + off; + struct fuse_folio_desc desc = { + .offset = off, + .length = len, + }; + struct fuse_io_args ia = { + .ap.args.out_pages = true, + .ap.num_folios = 1, + .ap.folios = &folio, + .ap.descs = &desc, + }; + ssize_t res; + + /* Don't overflow end offset */ + if (pos + (desc.length - 1) == LLONG_MAX) + desc.length--; + + fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) { + /* See fuse_do_readfolio() for why READ can return -EDEADLK */ + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) + res = AOP_TRUNCATED_PAGE; + return res; + } + + if (res < desc.length) + folio_zero_range(folio, off + res, desc.length - res); + + return 0; +} + +/** + * fuse_read_folio_merge - fill @folio without disturbing what it holds + * @file: file to read through + * @folio: the folio to fill + * + * iomap tracks a folio a block at a time, and a write that covered some + * of its blocks and not others leaves it valid in the ones it covered + * and not in the rest. The valid ones hold what that write put there, + * which writeback may not have sent yet; reading over them would lose + * it. Fetch the rest, in as few requests as the gaps allow. + * + * A folio the page cache tracks in one piece has no per block state to + * ask, and none to have: it is dirty only if a write covered it whole, + * and then it is valid and never reaches here. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_merge(struct file *file, struct folio *folio) +{ + size_t bsize = i_blocksize(folio->mapping->host); + size_t size = folio_size(folio); + size_t off = 0; + + while (off < size) { + size_t run = 0; + int err; + + /* Skip what the folio already holds */ + while (off < size && + iomap_is_partially_uptodate(folio, off, bsize)) + off += bsize; + + /* Take the gap behind it in one request */ + while (off + run < size && + !iomap_is_partially_uptodate(folio, off + run, bsize)) + run += bsize; + + if (!run) + break; + + err = fuse_read_folio_range(file, folio, off, run); + if (err) + return err; + off += run; + } + + return 0; +} + static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -1138,6 +1279,7 @@ static const struct iomap_read_ops fuse_iomap_read_ops = { static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_fill_read_data data = { .file = file, }; @@ -1145,7 +1287,6 @@ static int fuse_read_folio(struct file *file, struct folio *folio) .cur_folio = folio, .ops = &fuse_iomap_read_ops, .read_ctx = &data, - }; if (fuse_is_bad(inode)) { @@ -1153,54 +1294,68 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return -EIO; } + /* + * Writeback unlocks a folio as soon as it has handed it over, with + * the writeback flag still on it, so this can be reached while a + * FUSE_WRITE is still reading out of it. Filling it now would + * rewrite what is being sent, and past the end of the file the reply + * comes back short and zeroes it. Nothing reached here before a + * partial write started leaving folios invalid, because a dirty + * folio was always valid and never came this way. + */ + folio_wait_writeback(folio); + + /* + * Only a folio still holding what a write put in it has anything to + * keep. One that was merely being written back is clean by now, + * waited out just above, and reads whole. + * + * The gaps are read one run at a time rather than through iomap, + * which owns the whole folio and would fill what is kept here. + */ + if (fc->writeback_cache && folio_test_dirty(folio)) { + int err = fuse_read_folio_merge(file, folio); + + if (!err) + folio_mark_uptodate(folio); + + fuse_invalidate_atime(inode); + folio_unlock(folio); + return err; + } + iomap_read_folio(&fuse_iomap_ops, &ctx, NULL); fuse_invalidate_atime(inode); return 0; } +/* + * What iomap_file_buffered_write() carries for fuse, reached from the + * read-back callback as iter->private. @file is what that callback needs + * anyway, so the retry flag rides along and lives exactly as long as the + * call, with nothing to allocate or free. + */ +struct fuse_iomap_write_ctx { + struct file *file; + /* fuse_iomap_read_folio_range() hit AOP_TRUNCATED_PAGE */ + bool retry_needed; + /* + * The grant ->get_folio pinned, live until ->put_folio drops it. + * Here rather than on their stacks, which do not span the pair. + */ + struct fuse_dlm_span pin; +}; + static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct folio *folio, loff_t pos, size_t len) { - struct file *file = iter->private; - struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_iomap_write_ctx *ctx = iter->private; + struct file *file = ctx->file; size_t off = offset_in_folio(folio, pos); - bool hole; int ret; - /* - * Expanding writes claim their new i_size up front (see - * fuse_cache_write_iter()), which keeps iomap's own beyond-EOF - * zeroing in iomap_block_needs_zeroing() from ever firing for the - * write's own range: every block of a file expansion would be read - * from the server although it cannot contain data. Zero-fill - * locally instead when the server is known to hold no data in the - * range and we hold the DLM write lock covering it: - * - * - fi->server_size bounds the data materialized on the server - * (writeback and direct write acknowledgements, server - * attributes), - * - local data not yet acknowledged sits in uptodate blocks, which - * iomap never passes to this callback, - * - the page-granular DLM write lock excludes data written by - * other nodes, re-checked against the live lock tree so a - * revoked lock falls back to reading. - */ - if (fc->dlm) { - spin_lock(&fi->lock); - hole = pos >= fi->server_size; - spin_unlock(&fi->lock); - - if (hole && fuse_dlm_range_is_locked(fi, pos, pos + len - 1, - FUSE_PAGE_LOCK_WRITE)) { - folio_zero_range(folio, off, len); - return 0; - } - } - - ret = fuse_do_readfolio(file, folio, off, len); + ret = fuse_read_folio_range(file, folio, off, len); /* * TEMPORARY WORKAROUND for iomap write deadlock: @@ -1211,7 +1366,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * * However, iomap doesn't understand AOP_TRUNCATED_PAGE. * We need to: - * 1. Mark the retry flag (caller stored it in xarray) + * 1. Mark the retry flag on the caller's write context * 2. Convert to -EAGAIN so iomap sees an error * 3. Let fuse_cache_write_iter() detect and retry * @@ -1222,13 +1377,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. */ if (ret == AOP_TRUNCATED_PAGE) { - struct fuse_dlm_retry *retry; - unsigned long task_key = (unsigned long)current; - - retry = xa_load(&fc->dlm_retry_tasks, task_key); - if (retry) { - retry->retry_needed = true; - } + ctx->retry_needed = true; /* Convert to -EAGAIN for iomap */ ret = -EAGAIN; @@ -1324,26 +1473,57 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * Readahead fills the page cache past the range the reader locked, + * so take a DLM read grant over the whole window here too. Folios + * the server handed out no lock for are folios it will not revoke + * when a remote node writes them, and a later read would be served + * from stale cache. Take the grant before any folio is pulled off + * @rac, so the window is either fully covered or not populated. + * + * Speculative pages are not worth serving uncovered: on a failed + * request drop the window and let read_pages() clean up the folios + * left in @rac. A server without DLM support answers -ENOSYS and + * clears fc->dlm, which is not a failure. + * + * ->readahead is entered with every folio of the window already + * locked, and pulling one off @rac is what unlocks it, so the round + * trip is taken under those locks. See the readahead line of + * Documentation/filesystems/locking.rst. + * + * What keeps that from closing a cycle is the direction a revoke + * travels. This asks for a read grant on a range it holds nothing + * on, so the lock in the way is another node's, and the revoke that + * frees it is sent there. Nothing here has to run for this request + * to be answered, so the folios stay locked only for as long as the + * round trip. + * + * A window this already holds part of is the open edge: the query + * fails for the whole of it, so the request goes out while that part + * is still held, and a revoke for that part is sent here. Whether + * the server can answer while a revoke of its own is outstanding is + * not something this side can know. + */ + if (fc->writeback_cache && fc->dlm) { + int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), + readahead_length(rac), + FUSE_PAGE_LOCK_READ); + + if (err < 0 && err != -ENOSYS) + return; + } + iomap_readahead(&fuse_iomap_ops, &ctx, NULL); } static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); -/* - * Bound on re-requesting a revoked DLM grant before a cached read is - * served unlocked; see fuse_cache_read_iter(). - */ -#define FUSE_DLM_READ_RETRIES 3 - static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; - int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1361,65 +1541,35 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* - * Fence the cache-serving read against a NOTIFY invalidate so we never - * hand back a folio the server has just superseded. The gate read side - * is per-CPU cheap; the NOTIFY holds the write side with priority. - * Re-check the forced-DIO latch under it: if a storm latched us while we - * waited on a pending writer, reroute to direct like the buffered write - * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * A NOTIFY invalidate racing this read drops the folios it + * supersedes, so the read either misses and refetches or returns + * data that was current when it was copied. There is nothing to + * fence: unlike a write, a read leaves nothing behind that could + * reach the server under a grant it no longer holds. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; + if (fuse_inode_force_dio(inode)) { + size_t count = iov_iter_count(to); -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); - } /* - * The DLM lock was requested before entering the gate, and - * the NOTIFY invalidate we may just have waited on revokes - * locks under the gate write side. Re-check the grant here - * and re-request with the gate dropped, so a - * FUSE_DLM_WB_LOCK round trip never parks a pending - * invalidate behind our own gate hold. Once the check - * passes the lock cannot go away for the rest of the gate - * hold. A failed or unrecorded request falls through - * unlocked, as before: the retry is taken even then (the - * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would - * be re-requested forever. The retry is also bounded: a - * remote writer can revoke each successful grant before - * the gate is re-entered, and a reader-only inode has no - * force-DIO latch to end such a storm, so after - * FUSE_DLM_READ_RETRIES re-requests the read is served - * unlocked rather than looping without bound. + * A write that passed this same check just before the latch + * took hold dirtied the page cache after the notify dropped + * it, and a direct read does not look there. Send it first. */ - if (!lock_err && fc->dlm && tries-- > 0 && - !fuse_dlm_lock_is_held(fi, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ)) { - percpu_up_read(wb_sem); - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - goto retry; + if (count) { + res = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + if (res) + return res; } + return fuse_direct_read_iter(iocb, to); } res = generic_file_read_iter(iocb, to); - if (wb_sem) - percpu_up_read(wb_sem); - return res; } @@ -1492,15 +1642,6 @@ bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) spin_lock(&fi->lock); fi->attr_version = atomic64_inc_return(&fc->attr_version); - if (written > 0 && S_ISREG(inode->i_mode)) { - /* - * The server acknowledged data up to @pos, keep the - * server-materialized bound in sync for the expansion - * zero-fill in fuse_iomap_read_folio_range(). - */ - if (pos > fi->server_size) - fi->server_size = pos; - } if (written > 0 && pos > inode->i_size) { i_size_write(inode, pos); ret = true; @@ -1569,7 +1710,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, { struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); - unsigned offset = pos & (PAGE_SIZE - 1); size_t count = 0; unsigned int num; int err = 0; @@ -1596,7 +1736,13 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, if (mapping_writably_mapped(mapping)) flush_dcache_folio(folio); - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; + /* + * From @pos, not carried across iterations: a write landing + * inside a folio it does not start covers the rest of it, and + * a residue kept from that lands the next folio's copy past + * its end. + */ + folio_offset = offset_in_folio(folio, pos); bytes = min(folio_size(folio) - folio_offset, num); tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); @@ -1626,9 +1772,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, count += tmp; pos += tmp; num -= tmp; - offset += tmp; - if (offset == folio_size(folio)) - offset = 0; /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) @@ -1640,7 +1783,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, ia->write.folio_locked = true; break; } - if (!fc->big_writes || offset != 0) + /* + * Carry on only from a folio boundary: a copy that stopped + * short leaves the next one starting inside a folio, which is + * one request's worth on its own. + */ + if (!fc->big_writes || folio_offset + tmp != folio_size(folio)) break; } @@ -1656,7 +1804,8 @@ static inline unsigned int fuse_wr_pages(loff_t pos, size_t len, return min(pages, max_pages); } -static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) +static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii, + bool cache) { struct address_space *mapping = iocb->ki_filp->f_mapping; struct inode *inode = mapping->host; @@ -1686,6 +1835,14 @@ static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) if (count <= 0) { err = count; } else { + /* + * On behalf of a buffered write whose bytes bypass + * the page cache (DLM unaligned edges): the server + * must classify them like the writeback they + * replace. + */ + if (cache) + ia.write.in.write_flags |= FUSE_WRITE_CACHE; err = fuse_send_write_pages(&ia, iocb, inode, pos, count); if (!err) { @@ -1787,38 +1944,119 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) } } +/* + * How many times a writer confirms its grant again before giving up on + * the range. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation. + */ +#define FUSE_DLM_PIN_RETRIES 16 + +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + unsigned int tries = FUSE_DLM_PIN_RETRIES; + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (!tries--) + return -EIO; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + +/* + * The grant over the bytes about to be copied, held until ->put_folio + * hands the folio back dirty. Nothing is locked yet, which is what lets + * this wait out a revoke and ask for the grant again; ->writeback_range + * can do neither. + */ +static struct folio *fuse_iomap_get_folio(struct iomap_iter *iter, loff_t pos, + unsigned int len) +{ + struct fuse_iomap_write_ctx *ctx = iter->private; + struct folio *folio; + int err; + + err = fuse_dlm_pin_write(ctx->file, &ctx->pin, pos, len); + if (err) + return ERR_PTR(err); + + folio = iomap_get_folio(iter, pos, len); + if (IS_ERR(folio)) + fuse_dlm_unpin(get_fuse_inode(iter->inode)); + + return folio; +} + +/* iomap_write_end() has dirtied whatever was copied by now */ +static void fuse_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int copied, struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); + fuse_dlm_unpin(get_fuse_inode(inode)); +} + static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; +static const struct iomap_write_ops fuse_iomap_dlm_write_ops = { + .get_folio = fuse_iomap_get_folio, + .put_folio = fuse_iomap_put_folio, + .read_folio_range = fuse_iomap_read_folio_range, +}; + static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, struct iov_iter *from, struct file *file) { - struct fuse_conn *fc = get_fuse_conn(file_inode(file)); - ssize_t written, total_written = 0; - /* * TEMPORARY WORKAROUND for iomap write deadlock: * - * Stack-allocate retry state and register it before calling - * iomap. If fuse_iomap_read_folio_range() encounters - * AOP_TRUNCATED_PAGE, it will mark retry_needed. - * - * Stack allocation ensures no memory leaks - the state is - * valid for the duration of this function call and is - * automatically cleaned up. + * The context fuse_iomap_read_folio_range() marks when it hits + * AOP_TRUNCATED_PAGE. iomap hands it back through its private + * pointer, which fuse needs for @file either way. */ - struct fuse_dlm_retry retry_state = { - .retry_needed = false, - }; - unsigned long task_key = (unsigned long)current; - int xa_ret; - - xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, - &retry_state, GFP_KERNEL)); - if (xa_ret) - return xa_ret; + struct fuse_iomap_write_ctx ctx = { .file = file }; + struct fuse_conn *fc = get_fuse_conn(file_inode(file)); + ssize_t written, total_written = 0; retry: /* @@ -1829,14 +2067,16 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * the next iteration with iov_iter already drained, and iomap * would re-enter with len==0 and livelock on a 0-length mapping. */ - retry_state.retry_needed = false; + ctx.retry_needed = false; /* * Use iomap so that we can do granular uptodate reads * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, file); + fc->dlm ? &fuse_iomap_dlm_write_ops : + &fuse_iomap_write_ops, + &ctx); if (written > 0) total_written += written; @@ -1848,20 +2088,124 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * The folio has been unlocked by fuse_do_readfolio(), * breaking the ABBA deadlock with page invalidation. * - * Keep the entry in xarray and reuse it for the retry. - * * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE * retry support. */ - if (retry_state.retry_needed && iov_iter_count(from)) + if (ctx.retry_needed && iov_iter_count(from)) goto retry; - /* Remove from xarray now that we're done */ - xa_erase(&fc->dlm_retry_tasks, task_key); - return written < 0 ? written : total_written; } +/* + * Write @len bytes of @from at the current iocb position, either straight + * through to the server (@through, for an unaligned edge) or through the + * iomap page cache path (@through == false, for the aligned interior). + * Both primitives consume @len bytes from @from and advance iocb->ki_pos; + * the iterator is temporarily capped to @len so the unconsumed tail stays + * available for the next chunk. Returns bytes written (< @len means a + * short write, the caller stops) or a negative error. + */ +static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, + struct file *file, size_t len, bool through) +{ + struct fuse_dlm_span pin; + size_t hidden; + ssize_t res; + + if (!len) + return 0; + + /* Cap the iterator to this chunk, keeping the tail for later chunks. */ + hidden = iov_iter_count(from) - len; + iov_iter_truncate(from, len); + if (through) { + /* + * These bytes never enter the page cache, so a revoke cannot + * find them by flushing it. Hold the grant across the + * FUSE_WRITE instead, which is the reply the revoke already + * waits for when the same bytes go through writeback. + */ + res = fuse_dlm_pin_write(file, &pin, iocb->ki_pos, len); + if (!res) { + res = fuse_perform_write(iocb, from, true); + fuse_dlm_unpin(get_fuse_inode(file_inode(file))); + } + } else { + res = fuse_writeback_write_iter(iocb, from, file); + } + /* Restore from the iterator's own residue, so short writes/errors + * (which leave it partly advanced) reexpand to the exact remainder. */ + iov_iter_reexpand(from, iov_iter_count(from) + hidden); + + return res; +} + +/* + * Buffered write under DLM. A partly written page dirtied for + * writeback would have to be completed by reading the untouched + * remainder back from the server, and for a write past the server EOF + * that READ can only return zero bytes: a wasted round trip per + * unaligned edge. So cache only the page-aligned interior, whole pages + * need no read-modify-write, and route the unaligned head and tail + * straight through to the server. The writethrough path writes just + * those bytes and leaves the page non-uptodate, doing no read, and each + * edge lands as an independent FUSE_WRITE carrying FUSE_WRITE_CACHE + * like the writeback it replaces, so writers sharing a boundary page + * accumulate their bytes on the server. Aligned writes take the + * interior path whole; a sub-page write with no aligned interior goes + * fully through. + * + * The page is the block here: iomap tracks a folio a block at a time + * and __iomap_write_begin() skips the fill only for a block the write + * covers whole, so the cut has to land on block bounds. A writeback + * connection is refused unless the two are the same size; see + * process_init_reply(). + */ +static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, + struct iov_iter *from, + struct file *file) +{ + loff_t pos = iocb->ki_pos; + loff_t end = pos + iov_iter_count(from); + loff_t mid_start = round_up(pos, PAGE_SIZE); + loff_t mid_end = round_down(end, PAGE_SIZE); + /* Unaligned head, cached interior of whole pages, unaligned tail */ + const struct { + loff_t len; + bool through; + } chunk[] = { + { mid_start - pos, true }, + { mid_end - mid_start, false }, + { end - mid_end, true }, + }; + ssize_t res, total = 0; + unsigned int i; + + /* No whole page inside the write: nothing cacheable, all through. */ + if (mid_end <= mid_start) + return fuse_dlm_write_chunk(iocb, from, file, + iov_iter_count(from), true); + + /* + * Every chunk reports a failure the same way: the error while + * nothing has landed, a short write once something has. Returning + * what has landed when that is nothing reports no error and no + * bytes, which fuse_cache_write_iter() turns into the full count. + */ + for (i = 0; i < ARRAY_SIZE(chunk); i++) { + res = fuse_dlm_write_chunk(iocb, from, file, chunk[i].len, + chunk[i].through); + if (res < 0) + return total ? total : res; + total += res; + if (res < chunk[i].len) + break; + } + + return total; +} + static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); /* @@ -1908,19 +2252,13 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) * fc->dlm: the server has no DLM, proceed as a plain cached write. Any * other failure means the cache would be dirtied without DLM coverage - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed, but flag it - * so the in-gate re-validation skips a check an invisible grant could - * never pass. + * lock (positive return) is covered cluster-wide; proceed. */ -static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, - bool *unrecorded) +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) { int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); - if (err < 0 && err != -ENOSYS) - return err; - *unrecorded = err > 0; - return 0; + return (err < 0 && err != -ENOSYS) ? err : 0; } static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -1933,13 +2271,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; bool writeback = false; - bool wb_guard = false; - bool exclusive = true; - bool dlm_unrecorded = false; - loff_t dlm_pos = 0; - size_t dlm_len = 0; + bool exclusive; if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -1983,82 +2316,64 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) writeback = true; } - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); - /* * Request the DLM write lock before taking i_rwsem: the request is * an unbounded cluster round trip, and holding the writer-priority * rwsem across it would park a truncate -- and behind it every - * later writer -- for the duration. The grant-to-use window this - * leaves open is closed by the in-gate re-validation below. Only - * the append case must wait for the lock: its range depends on - * i_size, which is stable only under the exclusive inode lock. + * later writer -- for the duration. Only the append case must wait + * for the lock: its range depends on i_size, which is settled by + * generic_write_checks() under the exclusive inode lock. + * + * The request may find that the server has no DLM at all and clear + * fc->dlm, so pick the lock mode after it rather than before. The + * relaxed shared lock is only sound under DLM: the shared path + * claims the i_size extension up front, which stops iomap from + * zeroing beyond EOF, and the zero-fill that replaces it in + * fuse_iomap_read_folio_range() is itself gated on fc->dlm. Chosen + * too early, an expanding write would fall through to a READ of a + * range that cannot hold data -- which fails outright on a handle + * the client opened write-only. */ if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { - dlm_pos = iocb->ki_pos; - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, + iov_iter_count(from)); if (err) return err; - - /* - * The request above may have found that the server has no DLM - * at all, in which case it cleared fc->dlm. The relaxed shared - * lock was chosen just before, while fc->dlm still read 1, and - * it is only sound under DLM: the shared path claims the i_size - * extension up front, which stops iomap from zeroing beyond - * EOF, and the zero-fill that replaces it in - * fuse_iomap_read_folio_range() is itself gated on fc->dlm. - * Left as chosen, an expanding write would fall through to a - * READ of a range that cannot hold data -- which fails outright - * on a handle the client opened write-only. Re-decide now, - * while no lock is held yet. - */ - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* note that this small code dup will save us a lot of headache later - * when appends are done concurrently without using parallel direct writes */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { - /* - * An append write lands at the current EOF no matter what - * ki_pos holds: generic_write_checks() rewrites ki_pos to - * i_size for IOCB_APPEND, and i_size is stable here because - * append writes hold the inode lock exclusive. Lock where - * the data will land. - */ - dlm_pos = i_size_read(inode); - dlm_len = iov_iter_count(from); + err = count = generic_write_checks(iocb, from); + if (err <= 0) + goto out; - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + /* + * An append lands at the EOF generic_write_checks() has just written + * into ki_pos, not where the caller pointed, and the exclusive inode + * lock does not pin i_size either: attribute replies move it under + * fi->lock alone. Take the grant here, where the range is settled. + */ + if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; } - err = count = generic_write_checks(iocb, from); - if (err <= 0) - goto out; - /* - * Kill suid/sgid and stamp the timestamps here, before the gate, - * instead of leaving them next to the write itself. kiocb_modified() - * -> file_remove_privs() is the one that reaches the server: without - * handle_killpriv[_v2] fuse_setattr() kills the bits by asking it (a - * FUSE_GETATTR to refresh the mode, then a FUSE_SETATTR, which for a - * writeback inode first flushes and freezes writepages), and - * security_inode_killpriv() can drop the capability xattr with another - * round trip. A server may have to invalidate this inode from inside - * such a handler; its NOTIFY_INVAL_INODE then blocks in - * percpu_down_write() draining a gate reader that is itself waiting for - * the reply. Nothing held under the gate may wait for the server. + * Kill suid/sgid and stamp the timestamps here, ahead of the write + * itself. kiocb_modified() -> file_remove_privs() is the one that + * reaches the server: without handle_killpriv[_v2] fuse_setattr() + * kills the bits by asking it (a FUSE_GETATTR to refresh the mode, + * then a FUSE_SETATTR, which for a writeback inode first flushes and + * freezes writepages), and security_inode_killpriv() can drop the + * capability xattr with another round trip. A server may have to + * invalidate this inode from inside such a handler, and it must not + * find this write holding anything it needs. * * This also runs before the forced-DIO re-route below, so a re-routed * write repeats it; there is nothing left to do the second time. @@ -2067,31 +2382,32 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) goto out; - wb_guard = !!wb_sem; - if (wb_guard) { -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - fuse_cache_wr_unlock(inode, exclusive); - return fuse_direct_write_iter(iocb, from); - } - if (writeback && fc->dlm && !dlm_unrecorded && - !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, - FUSE_PAGE_LOCK_WRITE)) { - percpu_up_read(wb_sem); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) { - /* The gate is already dropped; funnel the - * failure through the one audited exit. */ - wb_guard = false; - goto out; - } - goto retry; - } + if (fuse_inode_force_dio(inode)) { + /* + * As on the read side, only worse: the direct write would + * land under whatever a write racing the latch left dirty, + * and the invalidate fuse_direct_write_iter() does after it + * launders rather than drops, putting that folio on the + * server on top. Send it first and the order is ordinary. + */ + if (count) + err = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + fuse_cache_wr_unlock(inode, exclusive); + if (err) + return err; + return fuse_direct_write_iter(iocb, from); } + /* + * A NOTIFY invalidate can revoke the grant requested above between + * here and the dirtying below, and nothing stops it: the bytes are + * caught on the way out instead. fuse_dlm_unlock_range() keeps a + * revoked range for as long as there is page cache under it, and + * writeback holds the range again before sending anything. So a + * write racing a revoke costs a round trip, not coverage. + */ + task_io_account_write(count); if (iocb->ki_flags & IOCB_DIRECT) { @@ -2099,7 +2415,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (written < 0 || !iov_iter_count(from)) goto out; written = direct_write_fallback(iocb, from, written, - fuse_perform_write(iocb, from)); + fuse_perform_write(iocb, from, + false)); } else if (writeback) { loff_t pos = iocb->ki_pos; loff_t end = pos + count; @@ -2130,17 +2447,53 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) spin_lock(&fi->lock); orig_size = i_size_read(inode); if (end > orig_size) { + /* + * Retire the attribute replies already on the + * wire. fuse_attr_cache_mask() decides whether + * the server's size wins from an i_size it read + * before this claim and before it slept in the + * lock tree query, so a GETATTR that left while + * i_size still matched the server's is applied + * afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() + * drop those replies, which is what + * fuse_write_update_attr() moves it for. + */ + fi->attr_version = + atomic64_inc_return(&fc->attr_version); + /* + * And count the claim, for a reply that leaves + * after it: until the bytes are dirtied, + * [orig_size, end) is covered by nothing else + * fuse_attr_cache_mask() can see. + */ + atomic_inc(&fi->size_extenders); i_size_write(inode, end); extended = true; } spin_unlock(&fi->lock); - /* Zero the tail of the folio straddling the old EOF. */ + /* + * Zero the tail of the folio straddling the old EOF. + * Inert while the fuse block size is PAGE_SIZE, which + * it always is, and nothing is recorded for it either + * way: claiming the whole gap as written would hand + * writeback bytes no one wrote. + */ if (extended && orig_size < pos) pagecache_isize_extended(inode, orig_size, pos); } - written = fuse_writeback_write_iter(iocb, from, file); + /* + * Under DLM the unaligned edges go through to the server + * instead of being completed by a read-modify-write READ + * (see fuse_dlm_buffered_write()); only whole blocks are + * cached for writeback. + */ + if (fc->dlm) + written = fuse_dlm_buffered_write(iocb, from, file); + else + written = fuse_writeback_write_iter(iocb, from, file); /* * Reconcile the speculative extension with what was actually @@ -2159,6 +2512,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) i_size_write(inode, reached); spin_unlock(&fi->lock); } + atomic_dec(&fi->size_extenders); } if (written < 0) { @@ -2166,11 +2520,9 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) goto out; } } else { - written = fuse_perform_write(iocb, from); + written = fuse_perform_write(iocb, from, false); } out: - if (wb_guard) - percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -2387,6 +2739,15 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (res > 0) *ppos = pos; + if (res > 0 && write && fopen_direct_io) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + invalidate_inode_pages2_range(mapping, idx_from, idx_to); + } + return res > 0 ? res : err; } EXPORT_SYMBOL_GPL(fuse_direct_io); @@ -2529,6 +2890,25 @@ static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, return iter_file_splice_write(pipe, out, ppos, len, flags); } +/* + * The folios and descs of a writeback request come from one allocation, + * which kfree(ap->folios) releases. + */ +static struct folio **fuse_wb_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **descs) +{ + struct folio **folios; + + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc)), flags); + if (!folios) + return NULL; + + *descs = (void *) (folios + nfolios); + + return folios; +} + static void fuse_writepage_free(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2542,6 +2922,25 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } +/* + * Ending a folio's writeback is iomap's own accounting, and fuse reports + * each queued run's bytes as its request completes. + * + * For a folio of more than one block, iomap_writeback_init() charges the + * whole folio to ifs->write_bytes_pending before any run is queued and + * iomap_writeback_folio() takes back what it did not submit once the last + * run has been offered, so the count cannot reach zero while a run is + * still to come and the completions sum to exactly the folio. + * + * For a folio of a single block there is no iomap_folio_state and + * iomap_finish_folio_write() ends the writeback on every call, but such a + * folio is offered as exactly one run: iomap_find_dirty_range() has no + * per-block state to walk and returns the whole range at once, and + * ->writeback_range never returns short, so iomap_writeback_range() does + * not go round again. One run is one entry in one request, and one call + * ends it. fuse_iomap_writeback_range() asserts that. + */ + static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2654,20 +3053,6 @@ static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, if (!fc->writeback_cache) fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); spin_lock(&fi->lock); - if (!error) { - struct fuse_write_in *inarg = &wpa->ia.write.in; - - /* - * The server acknowledged this writeback, so data up to the - * end of the request is materialized on the server. Advance - * the bound before the folios end writeback below, i.e. - * before they can go clean and be reclaimed, so that - * fuse_iomap_read_folio_range() can never zero-fill a - * reclaimed range the server holds data in. - */ - if ((loff_t) (inarg->offset + inarg->size) > fi->server_size) - fi->server_size = inarg->offset + inarg->size; - } fi->writectr--; fuse_writepage_finish(wpa); spin_unlock(&fi->lock); @@ -2718,7 +3103,7 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) if (wpa) { ap = &wpa->ia.ap; ap->num_folios = 0; - ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + ap->folios = fuse_wb_folios_alloc(1, GFP_NOFS, &ap->descs); if (!ap->folios) { kfree(wpa); wpa = NULL; @@ -2743,7 +3128,7 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, } static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, - uint32_t folio_index, loff_t offset, unsigned len) + uint32_t folio_index, loff_t offset, unsigned int len) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2778,16 +3163,44 @@ static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio return wpa; } +/* + * How many times a data integrity writeback goes round for runs it had to + * skip. Each pass takes the grants the one before it deferred, so one more + * is normally enough; the cap is there because a revoke can take them again. + */ +#define FUSE_WB_DEFER_PASSES 4 + struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; unsigned int max_folios; + /* + * The folio currently being split into runs, and whether any run of + * it has already been queued. Only a folio iomap tracks per block + * can be offered as more than one run; see fuse_writepage_finish(). + */ + struct folio *wb_folio; + bool wb_queued; /* * nr_bytes won't overflow since fuse_folios_need_send() caps * wb requests to never exceed fc->max_pages (which has an upper bound * of U16_MAX). */ unsigned int nr_bytes; + /* + * The runs this pass could not send because their grant had gone. + * Taken back in fuse_iomap_writeback_submit(), where no folio is + * held, for the pass that follows; see fuse_iomap_writeback_range(). + */ + u64 regrant_start; + u64 regrant_end; + /* fuse_iomap_writeback_submit() took that range back */ + bool regranted; + /* + * A folio whose run was skipped, held by a reference until it can be + * put back on the dirty list; see fuse_writeback_redirty(). + */ + struct folio *redirty; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, @@ -2802,7 +3215,7 @@ static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, max_pages); WARN_ON(nfolios <= data->max_folios); - folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + folios = fuse_wb_folios_alloc(nfolios, GFP_NOFS, &descs); if (!folios) return false; @@ -2857,6 +3270,95 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, return false; } +/* + * A server that asked for an alignment wants its writes to start on it. Close + * the run at an aligned position when the next aligned run cannot be reached, + * either because writeback ends before it or because it would not fit. + */ +static bool fuse_writeback_reached_alignment(struct fuse_conn *fc, loff_t pos, + unsigned int bytes, + struct writeback_control *wbc) +{ + unsigned int total_pages = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; + pgoff_t page_index = pos >> PAGE_SHIFT; + pgoff_t end_page_index; + + if (!fc->alignment_pages) + return false; + + if (page_index % fc->alignment_pages) + return false; + + /* + * fuse_launder_folio() has no writeback_control, and a folio of more + * than one run reaches here from it. + */ + if (!wbc) + return false; + + end_page_index = (wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (page_index + fc->alignment_pages > end_page_index) + return true; + + return total_pages + fc->alignment_pages > fc->max_pages; +} + +/* + * Put a folio writeback could not send back on the dirty list. + * + * iomap takes the dirty flag off a folio before it offers it to + * ->writeback_range, and a run reporting an error, or reporting a hole + * because the grant has gone, has thrown its bytes away unless they are put + * back. + * + * Not from inside the callback, though. iomap_writeback_folio() runs + * iomap_clear_range_dirty() over the whole folio once that has returned, so + * a range put back there is wiped again. For a folio one block wide that + * call does nothing and it would not matter, but a large folio carries an + * iomap_folio_state, and the folio would then be left with the dirty flag + * and no dirty block under it: the next pass finds nothing to write and the + * folio goes clean with its bytes never sent. + * + * So hold the folio and dirty it once iomap has let go of it, which on the + * ->writepages path is the next call or the submit, and in + * fuse_launder_folio() is the submit it makes itself. + * + * Only while there is a connection left to take the bytes. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty_done(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc) +{ + struct folio *folio = data->redirty; + + if (!folio) + return; + data->redirty = NULL; + + if (READ_ONCE(fc->connected)) { + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); + } + folio_put(folio); +} + +/* Remember @folio for fuse_writeback_redirty_done() */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc, + struct folio *folio) +{ + if (data->redirty == folio) + return; + + fuse_writeback_redirty_done(fc, data, wbc); + folio_get(folio); + data->redirty = folio; +} + static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct folio *folio, u64 pos, unsigned len, u64 end_pos) @@ -2867,15 +3369,153 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct inode *inode = wpc->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - loff_t offset = offset_in_folio(folio, pos); + struct fuse_dlm_span pin; + bool pinned = false; + loff_t offset; WARN_ON_ONCE(!data); if (!data->ff) { data->ff = fuse_write_file_get(fi); - if (!data->ff) + if (!data->ff) { + /* + * No file left open for writing, which + * fuse_open()'s invalidate reaches through + * fuse_launder_folio() once the last writer has + * closed. The bytes are still the newest there + * are, so keep them: dropping them here loses a + * write that fsync and close both reported done. + */ + fuse_writeback_redirty(fc, data, wpc->wbc, folio); return -EIO; + } + } + + /* + * A folio iomap has not asked about before: the one before it has all + * of its runs queued. + */ + if (data->wb_folio != folio) { + data->wb_folio = folio; + data->wb_queued = false; + /* + * iomap has unlocked whatever it offered before this, so a + * folio held from then can go back on the dirty list now. + */ + fuse_writeback_redirty_done(fc, data, wpc->wbc); + } + + /* + * Every run iomap reports dirty was written whole by this client: + * the unaligned edges of a cached write go to the server directly + * and the interior covers whole blocks, so nothing partly written + * is ever dirtied. There is nothing to classify, only the grant to + * make sure of: a revoke may have arrived since the write, and + * these bytes must not go out from under one. + * + * fuse_dlm_regrant_range() takes the range back when it has gone, + * and walks the record once under the lock held for read when it + * has not. A failure redirties the folio, so the next writeback + * tries again. A revoke handler driving this skips the run + * entirely rather than ask for the grant it is taking away. + * + * The grant is pinned over this run from the moment it is found + * until the folio is under writeback, where the revoke waits for it + * again. Without that these bytes could still go out under a + * revoke that has already been answered. + */ + if (fc->dlm && fc->writeback_cache) { + int err; + + /* + * wpc->iomap.type carries over from the previous run and from + * the previous folio, so anything that is queued has to say + * so. Left at a stale IOMAP_HOLE, iomap takes the folio for + * one it never queued and ends its writeback while the write + * is still in flight. + */ + wpc->iomap.type = IOMAP_MAPPED; + + /* + * The revoke handler flushing the range it is taking + * away. That lock is still this client's until the + * handler returns, so send without asking: the record has + * gone already and asking would be a round trip for the + * very range being revoked. Only for that range, since + * the latched path launders the whole mapping and the + * rest of it may be covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + + /* + * The folio is locked and under writeback here, so a grant + * this run does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. read_folio() has AOP_TRUNCATED_PAGE to unlock and + * retry with; ->writeback_range has nothing of the sort. + * + * Report the run as a hole instead. The folio goes back on + * the dirty list, the range is remembered for + * fuse_iomap_writeback_submit() to take back with no folio + * held, and the pass that follows sends it. Nothing is lost + * and no error is recorded for a later fsync to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the run in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. + */ + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + u64 s = min(data->regrant_start, pos); + u64 e = max(data->regrant_end, pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips runs a long way apart, and the + * span between them says nothing about what + * is wanted; the runs left out stay dirty and + * a later pass asks for them. + */ + if (e - s <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = s; + data->regrant_end = e; + } + } + wpc->iomap.type = IOMAP_HOLE; + return len; + } + + /* + * Held, and pinned so it stays held: this walks the record + * and sends nothing. It stays a call rather than the check + * above so a grant that arrives between them is still used. + */ + err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); + if (err < 0 && err != -ENOSYS) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + fuse_dlm_unpin(fi); + return err; + } } +queue: + + offset = offset_in_folio(folio, pos); if (wpa) { bool send = fuse_folios_need_send(fc, pos, len, ap, @@ -2890,6 +3530,10 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, !fuse_pages_realloc(data, fc->max_pages); } + if (!send) + send = fuse_writeback_reached_alignment(fc, pos, + data->nr_bytes + len, wpc->wbc); + if (send) { fuse_writepages_send(inode, data); data->wpa = NULL; @@ -2899,15 +3543,36 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (data->wpa == NULL) { wpa = fuse_writepage_args_setup(folio, offset, data->ff); - if (!wpa) + if (!wpa) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + if (pinned) + fuse_dlm_unpin(fi); return -ENOMEM; + } fuse_file_get(wpa->ia.ff); data->max_folios = 1; ap = &wpa->ia.ap; } - fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, - offset, len); + /* + * fuse_writepage_finish() reports this run's bytes when the request + * completes. A folio of a single block carries no iomap_folio_state + * and that report ends its writeback outright, which is only correct + * because iomap offers such a folio as exactly one run. Nothing + * above can make it offer a second, but the accounting depends on it. + */ + WARN_ON_ONCE(data->wb_queued && i_blocks_per_folio(inode, folio) == 1); + data->wb_queued = true; + + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, offset, len); data->nr_bytes += len; ap->num_folios++; @@ -2924,11 +3589,34 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, WARN_ON_ONCE(!data); + /* No more runs are coming for the folio last seen */ + data->wb_folio = NULL; + data->wb_queued = false; + fuse_writeback_redirty_done(get_fuse_conn(wpc->inode), data, wpc->wbc); + if (data->wpa) { WARN_ON(!data->wpa->ia.ap.num_folios); fuse_writepages_send(wpc->inode, data); } + /* + * Take back what the runs above had to skip, so the pass that + * follows finds the grant and sends the folios they left dirty. + * + * Only on the ->writepages path, which is what @wbc marks. + * fuse_launder_folio() reaches here with the folio still locked by + * folio_unmap_invalidate(), which is the ordering this is avoiding, + * and a revoke handler would ask for the very range it is revoking. + * In both the skip simply stands and writeback picks it up. + */ + if (wpc->wbc && data->ff && data->regrant_end > data->regrant_start && + !fuse_in_notify_ctx()) { + fuse_dlm_regrant_range(data->ff, wpc->inode, + data->regrant_start, + data->regrant_end - 1); + data->regranted = true; + } + if (data->ff) fuse_file_put(data->ff, false); @@ -2945,14 +3633,8 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data = {}; - struct iomap_writepage_ctx wpc = { - .inode = inode, - .iomap.type = IOMAP_MAPPED, - .wbc = wbc, - .ops = &fuse_writeback_ops, - .wb_ctx = &data, - }; + unsigned int tries = FUSE_WB_DEFER_PASSES; + int err; if (fuse_is_bad(inode)) return -EIO; @@ -2961,7 +3643,35 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - return iomap_writepages(&wpc); + /* + * A run whose grant had gone is skipped and its range taken back in + * fuse_iomap_writeback_submit(), which leaves the folio dirty for a + * later pass. For a data integrity writeback there is no later + * pass: fsync() and close() would report the bytes written while + * they are still only in the page cache. Go round again, now that + * the grant is held, until nothing is left deferred. + */ + do { + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = inode, + .iomap.type = IOMAP_MAPPED, + .wbc = wbc, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + err = iomap_writepages(&wpc); + /* + * Only where the submit took the grant back. A revoke + * handler driving this is not allowed to, so the runs it + * skipped would be skipped again by every pass. + */ + if (err || wbc->sync_mode != WB_SYNC_ALL || !data.regranted) + break; + } while (--tries); + + return err; } static int fuse_launder_folio(struct folio *folio) @@ -3048,6 +3758,7 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l fuse_abort_conn(fc); err = -EINVAL; } + return err; } /* @@ -3073,9 +3784,13 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct fuse_mount *fm = get_fuse_mount(inode); if (fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; - size_t length = PAGE_SIZE; - int err = fuse_get_page_mkwrite_lock(file, pos, length); + /* + * The whole folio is dirtied on the way out of this fault + * (fault_dirty_shared_page()), so the lock has to cover the + * folio, not the page that faulted. + */ + int err = fuse_get_page_mkwrite_lock(file, folio_pos(folio), + folio_size(folio)); if (err < 0) { return vmf_error(err); } @@ -3123,7 +3838,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) /* * If the inode was latched into forced direct IO after a remote-modify * notification, a mapping needs the page cache, so revert to caching - * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * mode. Revert without the inode lock: ->mmap runs * under mmap_lock and the buffered write path holds both across a fault * on the user buffer (which takes mmap_lock), so taking either here * would invert lock order (ABBA). Clearing the latch and dropping the @@ -3986,28 +4701,11 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fuse_dlm_cache_init(fi); fi->writectr = 0; fi->iocachectr = 0; - fi->server_size = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - /* - * Coherency gate for the forced-direct-IO feature; only writeback+dlm - * regular files need it. A percpu_rw_semaphore embeds per-CPU state, - * so allocate it out of line and only when the mount can use it rather - * than paying it on every inode. On failure leave it NULL: the gate - * stays inactive (best-effort invalidate) and the inode is still usable. - */ - fi->wb_inval_rwsem = NULL; - if (fc->writeback_cache && fc->dlm) { - struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); - - if (sem && percpu_init_rwsem(sem)) { - kfree(sem); - sem = NULL; - } - fi->wb_inval_rwsem = sem; - } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; + atomic_set(&fi->size_extenders, 0); if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 6531186d63b54b..d3ef8548d3b380 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -1,599 +1,780 @@ // SPDX-License-Identifier: GPL-2.0-only /* * FUSE page lock cache implementation + * + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is + * cached there, and whether the server has seen it, is what the page + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include +#include #include #include -#include +#include +#include -/* A range of pages with a lock */ +/* + * How often to ask again for a grant a revoke killed while it was in + * flight, or the server refused as contended, before giving up on the + * range. Each pass is a round trip. + */ +#define FUSE_DLM_GRANT_RETRIES 16 + +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) + +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { - /* Interval tree node */ - struct rb_node rb; - /* Start page offset (inclusive) */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; - /* End page offset (inclusive) */ uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lock mode */ - enum fuse_page_lock_mode mode; - /* Temporary list entry for operations */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ + bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; + /* The cache->pending link */ struct list_head list; }; -/* Lock modes for FUSE page cache */ -#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ -#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ - -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) -{ - return range->start; -} - -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return range->end; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); - /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at * - * Initialize a page cache lock manager for a FUSE inode. + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. * - * Return: 0 on success, negative error code on failure + * Return: the shard, or NULL if it could not be allocated. */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; - if (!cache) - return -EINVAL; + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; - init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; - cache->revoke_gen = 0; + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; - return 0; + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } + + return shard; } /** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up + * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] + * @cache: The page cache + * @start: Start byte offset of the revoked region + * @end: End byte offset of the revoked region * - * Release all locks and free all resources associated with the cache. + * A revoke overlapping a request still on the wire has nothing to remove + * from the tree, since that grant is not recorded yet. Marking it makes + * fuse_dlm_request_commit() drop the grant instead of recording it. + * + * The nodes are owned by the threads waiting on their replies: mark + * only, never remove or free. + * + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ -void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, + uint64_t start, uint64_t end) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; - - if (!cache) - return; - - /* Release all locks */ - down_write(&cache->lock); - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); + struct fuse_dlm_range *req; + + spin_lock(&cache->pending_lock); + list_for_each_entry(req, &cache->pending, list) { + if (req->start <= end && start <= req->end) + req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; } - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * - * Return: Pointer to the first overlapping range, or NULL if none found + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_cache_init(struct fuse_inode *inode) { - return fuse_page_it_iter_first(&cache->ranges, start, end); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + init_rwsem(&cache->lock); + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); + INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); } -/** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = current; +} - if (!cache) - return; +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. + */ +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; - /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. - */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; - /* Try to merge with next range if adjacent and same mode */ - if (next && range->mode == next->mode && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } + return false; +} - /* Move to next range */ - range = next; - } +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) +{ + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; } /** - * __fuse_dlm_lock_range - Lock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode (read or write) - * @genp: If non-NULL, the revocation generation sampled before the grant - * was requested; recording fails with -EAGAIN if it has moved - * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * fuse_dlm_pin - hold the grants over a range until fuse_dlm_unpin() + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes * - * Return: 0 on success, negative error code on failure + * Waits out a revoke overlapping that range, so it must not be called + * with a folio held: the revoke drops that same page cache once it has + * drained. A revoke elsewhere in the file is not waited for. The caller + * confirms its grant after this returns, never before; a confirmation + * from before the pin says nothing. */ -static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - const uint64_t *genp) +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - int lock_mode; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; - if (!cache || start > end) - return -EINVAL; - - /* Convert to lock mode */ - lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; + /* + * A revoke handler driving this inode's page cache: the lock over + * the range is still this client's until the handler returns, and + * the fence this would wait on is the handler's own. Both ends + * test the same task, so nothing is left on the list. + */ + if (fuse_in_notify_ctx()) + return; - down_write(&cache->lock); + fuse_dlm_span_set(pin, offset, length); + spin_lock(&cache->pin_lock); + while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { + spin_unlock(&cache->pin_lock); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); + spin_lock(&cache->pin_lock); + } /* - * A revoke was processed after @genp was sampled; the grant this - * record carries may be the very one it targeted (a revoke of a - * not-yet-recorded grant removes nothing and would never be - * retried). Refuse, the caller re-requests. + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. */ - if (genp && cache->revoke_gen != *genp) { - up_write(&cache->lock); - return -EAGAIN; - } + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); +} - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check lock compatibility */ - if (lock_mode == FUSE_PCACHE_LK_WRITE && - lock_mode != range->mode) { - /* we own the lock but have to update it. */ - list_add_tail(&range->list, &to_upgrade); - } - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } +/** + * fuse_dlm_trypin - fuse_dlm_pin() for a caller that cannot sleep + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * For the writeback path, which holds a folio locked and under writeback + * and has nothing to wait with. A refusal means a revoke of this range + * is draining; the caller redirties and the pass that follows sends the + * folio. + * + * Return: true if the range is pinned, false if it is not. + */ +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return true; - /* Move to next range */ - range = next; - } + fuse_dlm_span_set(pin, offset, length); - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); - new_range->start = current_start; - new_range->end = end; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); + return !fenced; +} - list_add_tail(&new_range->list, &to_lock); - } +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; - /* update locks, if any lock is in this list it has the wrong mode */ - list_for_each_entry(range, &to_upgrade, list) { - /* Update the lock mode */ - range->mode = lock_mode; - } + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return; - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; + } } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); + if (waiters) + wake_up_all(&cache->pin_wq); +} - up_write(&cache->lock); - return 0; +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; - /* Restore original lock modes for any partially upgraded locks */ - list_for_each_entry(range, &to_upgrade, list) { - if (lock_mode == FUSE_PCACHE_LK_WRITE) { - /* We upgraded this lock but failed later, downgrade it back */ - range->mode = FUSE_PCACHE_LK_READ; - } - } + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); - up_write(&cache->lock); - return ret; + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); } -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) { - return __fuse_dlm_lock_range(inode, start, end, mode, NULL); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); + + wake_up_all(&cache->pin_wq); } -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen) +/** + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of + * + * Release all locks and free all resources associated with the cache. + */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { - return __fuse_dlm_lock_range(inode, start, end, mode, &gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_shard *shard; + unsigned long idx; + + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); + + xa_for_each(&cache->shards, idx, shard) + kfree(shard); + /* + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. + */ + xa_destroy(&cache->shards); + up_write(&cache->lock); } /** - * fuse_dlm_revoke_gen - sample the revocation generation - * @inode: the fuse inode + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in + * + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. + * + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. * - * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The - * reply and a NOTIFY revoke can be serviced on different threads, so a - * revoke may be processed between the reply arriving and its grant - * being recorded. fuse_dlm_lock_range_gen() re-checks the generation - * under the cache lock and refuses to record a grant such a revoke may - * have already killed. + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - return READ_ONCE(inode->dlm_locked_areas.revoke_gen); + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); + + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); + } } /** - * fuse_dlm_punch_hole - Punch a hole in a locked range - * @cache: The page cache - * @start: Start page offset of the hole - * @end: End page offset of the hole + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in * - * Create a hole in a locked range by splitting it into two ranges. + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. * - * Return: 0 on success, negative error code on failure + * Caller holds @cache->lock for read. + * + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_range *range, *new_range; - int ret = 0; + unsigned long idx, last_idx; - if (!cache || start > end) + if (start > end) return -EINVAL; - /* Find a range that contains [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - if (!range) { - ret = -EINVAL; - goto out; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* If the hole is at the beginning of the range */ - if (start == range->start) { - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - /* If the hole is at the end of the range */ - if (end == range->end) { - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; - /* The hole is in the middle, need to split */ - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out; + fuse_dlm_shard_record(shard, lo, hi, mode); } - /* Copy properties from original range, keeping the original end */ - *new_range = *range; - INIT_LIST_HEAD(&new_range->list); - new_range->start = end + 1; + return 0; +} +/** + * fuse_dlm_request_begin - publish a lock request before it is sent + * @inode: the fuse inode + * @req: caller-owned storage for the request, live until commit or abort + * @start: start byte offset being requested (inclusive) + * @end: end byte offset being requested (inclusive) + * + * The mode is not recorded here: until the server answers the range is + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). + * + * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different + * threads, so a revoke can be processed before the grant the reply + * carries is recorded. Publishing the request before it leaves gives + * that revoke a node to mark; without one it removes nothing, and the + * grant recorded afterwards is never taken back by any later NOTIFY. + * + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. + * + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + req->start = start; + req->end = end; /* - * Shorten the original end only while it is unlinked. range->end is - * used in calulating the interval tree's __subtree_end, so changes - * made while the node is still in the tree leaves every ancestor - * stale. Update range->end after fuse_page_it_remove() + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(new_range, &cache->ranges); - -out: - return ret; + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; + req->killed = false; + req->clamp = false; + + spin_lock(&cache->pending_lock); + list_add_tail(&req->list, &cache->pending); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_unlock_range - Unlock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_request_commit - retire a request and record its grant + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode that was requested * - * Release locks on the specified range of pages. An inverted range is - * rejected rather than silently removing nothing: the callers revoke - * coverage, and a revoke that quietly keeps the grant alive would let - * the re-validating IO paths trust a lock the server has taken away. - * To drop every grant use fuse_dlm_cache_release_locks() (there is no - * in-band sentinel range for it). + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. * - * Return: 0 on success, negative error code on failure + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. + * + * @req is retired in every case and may be reused. + * + * Return: -EAGAIN if a revoke overlapped the range @req asked for, + * nothing recorded; otherwise the result of recording the grant. */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; + bool revoked, clamp; int ret = 0; - if (!cache || start > end) - return -EINVAL; - - down_write(&cache->lock); - /* - * Unconditional, even when nothing overlaps: the revoke racing - * with an in-flight grant finds an empty tree precisely because - * the grant is not recorded yet, and the bump is what makes the - * recording side notice (see fuse_dlm_lock_range_gen()). + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. */ - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check if we need to punch a hole */ - if (start > range->start && end < range->end) { - /* Punch a hole in the middle */ - ret = fuse_dlm_punch_hole(cache, start, end); - if (ret) - goto out; - /* After punching a hole, we're done */ - break; - } else if (start > range->start) { - /* Adjust the end of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - } else if (end < range->end) { - /* Adjust the start of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - } else { - /* Complete overlap, remove the range */ - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } + down_read(&cache->lock); - range = next; + spin_lock(&cache->pending_lock); + list_del(&req->list); + revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; } + spin_unlock(&cache->pending_lock); -out: - up_write(&cache->lock); - return ret; + if (!revoked) + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); + + return revoked ? -EAGAIN : ret; } /** - * fuse_dlm_range_is_locked - Check if a page range is already locked - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode to check for (or NULL to check for any lock) + * fuse_dlm_request_abort - retire a request that got no usable reply + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * + * Nothing is recorded, so a mark left by a revoke does not matter. + */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pending_lock); + list_del(&req->list); + spin_unlock(&cache->pending_lock); +} + +/** + * fuse_dlm_unlock_range - Revoke the grants over a range of pages + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset * - * Check if the specified range of pages is already locked. - * The entire range must be locked for this to return true. + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. * - * Return: true if the entire range is locked, false otherwise + * An inverted range is rejected rather than silently revoking nothing: + * the callers revoke coverage, and a revoke that quietly keeps the grant + * alive would let the re-validating IO paths trust a lock the server has + * taken away. To drop every grant use fuse_dlm_cache_release_locks() + * (there is no in-band sentinel range for it). + * + * Return: 0 on success, negative error code on failure */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - int lock_mode = 0; - uint64_t current_start = start; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) - return false; + if (start > end) + return -EINVAL; - /* Convert to lock mode if specified */ - if (mode == FUSE_PAGE_LOCK_READ) - lock_mode = FUSE_PCACHE_LK_READ; - else if (mode == FUSE_PAGE_LOCK_WRITE) - lock_mode = FUSE_PCACHE_LK_WRITE; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); - down_read(&cache->lock); + /* + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. + */ + fuse_dlm_kill_pending(cache, start, end); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; - /* Check if the entire range is covered */ - while (range && current_start <= end) { /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. */ - if (lock_mode && range->mode < lock_mode) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } - - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; + /* + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. + */ + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); } - - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); } - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } + up_write(&cache->lock); + return 0; +} - up_read(&cache->lock); - return true; +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. + * + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; } /** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode + * fuse_dlm_range_is_locked - Check if a byte range is already locked + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * @mode: Lock mode to check for * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. - * - * Return: true if at least one recorded range is held for write + * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx, last_idx; + bool covered = true; + if (start > end) + return false; + + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->mode == FUSE_PCACHE_LK_WRITE) { - held = true; + + last_idx = end >> FUSE_DLM_SHARD_SHIFT; + + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -637,11 +818,10 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, * re-validating the grant must not re-request on a nonzero return or * they would spin. */ -int fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) { - struct fuse_file *ff = file->private_data; - struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; @@ -649,13 +829,24 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; - uint64_t gen; + struct fuse_dlm_range req; + uint64_t pg_start, pg_end; + uint64_t grant_start, grant_end; + int tries = FUSE_DLM_GRANT_RETRIES; int err; /* An empty range needs no lock. */ if (!length) return 0; + /* + * note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests + */ + pg_start = (uint64_t)offset & PAGE_MASK; + pg_end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected @@ -664,27 +855,20 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * The early exit uses the same helper the callers re-validate * with, so this check and a later fuse_dlm_lock_is_held() can * never disagree about what counts as covered. */ - if (fuse_dlm_lock_is_held(fi, offset, length, mode)) - return 0; /* we already have this area locked */ - - /* - * Sample the revocation generation before the request leaves. - * The reply and a NOTIFY revoke are serviced on different - * threads, so a revoke aimed at the grant this request returns - * can be processed before the grant is recorded below -- - * recording it anyway would resurrect a dead grant that no later - * NOTIFY will ever remove. - */ - gen = fuse_dlm_revoke_gen(fi); + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) { + /* + * Already covered, and the record says nothing beyond that, + * so this is one shared acquisition end to end. + */ + return 0; + } memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned - * here but since we only get here on writeback caching we will - * send out page aligned requests */ - inarg.start = offset & PAGE_MASK; - inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.start = pg_start; + inarg.end = pg_end; inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -696,18 +880,33 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; + + /* Publish before sending; see fuse_dlm_request_begin() */ + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end); + err = fuse_simple_request(fm, &args); - if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; + if (err) { + fuse_dlm_request_abort(fi, &req); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + return err; + } + /* + * The range is contended, the same answer a READ gets and + * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * caller to retry. There is no such convention here, and + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. + */ + if (err == -EDEADLK || err == -EAGAIN) + goto retry; return err; } - if (err) - return err; - if (inarg.start < outarg.start || inarg.end > outarg.end) { /* fuse server is seriously broken */ + fuse_dlm_request_abort(fi, &req); pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); @@ -715,22 +914,45 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, } /* - * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. + */ + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. + */ + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. */ - err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* Retire the request and record the grant */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); if (err == -EAGAIN) { /* - * A revoke was processed while the request was in flight; - * the grant may already be dead, so re-request instead of - * recording it. Retry until a grant survives long enough to - * be recorded: giving up here would hand the caller an error - * for a range no one else holds, and the write path turns - * that into a failed write. Each pass makes a fresh server - * round trip, so a revoke storm throttles this loop rather - * than spinning it. + * A revoke overlapping this range was processed while the + * request was in flight, so the grant is dead. Retry + * rather than fail: no one else holds the range, and the + * write path turns an error into a failed write. */ - goto restart; + goto retry; } /* @@ -745,4 +967,47 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, return FUSE_DLM_GRANT_UNRECORDED; return 0; + +retry: + /* + * Ask again, but not forever. Every pass is a whole round trip, + * which throttles the loop but does not end it, and writeback asks + * for a grant with a folio locked, so a node taking the range as + * fast as this asks for it would hold that folio and this task for + * as long as it kept going. + */ + if (fatal_signal_pending(current)) + return -EINTR; + if (!tries--) + return -EIO; + goto restart; +} + +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return __fuse_get_dlm_lock(file->private_data, file_inode(file), + offset, length, mode); +} + +/** + * fuse_dlm_regrant_range - hold [start, end] again for writeback + * @ff: a fuse file open for writing on @inode + * @inode: the inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * Writeback holds the range again before sending a folio, since a revoke + * may have arrived between the write and the send. Whatever the other + * holder wrote in between is overwritten, which for two writers that + * never synchronised is a legitimate order. + * + * A range still held is the ordinary case: the grant is found recorded + * and nothing is sent to the server. + */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end) +{ + return __fuse_get_dlm_lock(ff, inode, start, end - start + 1, + FUSE_PAGE_LOCK_WRITE); } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 30fdbb26bd3daf..ec2e82c406a1cb 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,16 +7,37 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include +#include struct fuse_inode; +struct fuse_dlm_range; +struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: a + * writer between confirming a grant and dirtying under it (@owner set), + * or a revoke taking grants away (@owner NULL). Caller-owned storage, + * live until the matching unpin or revoke end. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -26,53 +47,189 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 -/* Page cache lock manager */ +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + +/* + * Page cache lock manager. + * + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. + */ struct fuse_dlm_cache { - /* Lock protecting the tree */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of locked ranges */ - struct rb_root_cached ranges; /* - * Bumped under @lock by every revocation - * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); - * lets fuse_get_dlm_lock() order recording a reply's grant - * against revokes processed while the reply was in flight. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. */ - uint64_t revoke_gen; + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. + */ + struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * Writers between confirming a grant and dirtying under it, each + * over the range it is about to write. See the pin comment above. + */ + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a + * revoke processed while the reply is on the wire can mark it. @req is + * caller-owned storage, live until the matching commit or abort. The + * mode is not recorded until the grant is, so only the commit takes it. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end); -/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen); +/* + * Retire @req and record the grant [start, end] as one step under the + * cache lock. -EAGAIN means a revoke overlapped @req in flight and + * nothing was recorded; the caller must request again. @req is retired + * either way. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); -/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); +/* Retire @req without recording anything */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req); /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); + +/* + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* Hold [start, end] again so writeback can send what it found revoked */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c18ded6cd394a6..3abc44bc760038 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,8 +32,71 @@ #include #include #include +#include #include "fuse_dlm_cache.h" +/* + * Page cache work driven by a NOTIFY invalidate is marked on the task. + * + * Writeback reached from there must not ask the server for a grant: the + * range is the one the server is revoking, and the request would go out + * from inside the handler the server is waiting on, with the folio locked + * and under writeback. See fuse_reverse_inval_inode() and + * fuse_iomap_writeback_range(). + */ +extern const char fuse_notify_ctx_key[]; + +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) +{ + void *old = current->journal_info; + + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; + return old; +} + +static inline void fuse_notify_ctx_leave(void *old) +{ + current->journal_info = old; +} + +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + +static inline bool fuse_in_notify_ctx(void) +{ + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; +} + /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -159,6 +222,15 @@ struct fuse_inode { /** Time in jiffies until the file attributes are valid */ u64 i_time; + /* + * Time in jiffies until mode/uid/gid (the permission-check subset of + * STATX_BASIC_STATS) are valid. Tracked separately from i_time so that + * a partial statx refresh covering only the perm bits can extend the + * permission-check cache without falsely advancing i_time for the + * other (un-refreshed) attributes. + */ + u64 i_perm_time; + /* Which attributes are invalid */ u32 inval_mask; @@ -200,35 +272,6 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; - /* - * Server-materialized size: an upper bound for how far - * the server holds file data. Seeded from - * server-reported attributes, advanced when the server - * acknowledges data (writeback completion, - * fuse_write_update_attr()), lowered again on - * truncate. A read-modify-write of a block starting - * at or past this bound needs no READ request under a - * held DLM write lock: the server has no data there - * (see fuse_iomap_read_folio_range()). Protected by - * fi->lock. - */ - loff_t server_size; - - /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. - */ - struct percpu_rw_semaphore *wb_inval_rwsem; - /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations * for this whole file: notify_stamp is the jiffies of @@ -241,6 +284,20 @@ struct fuse_inode { */ unsigned long notify_stamp; unsigned int notify_interval_ewma; + + /* + * Buffered writes that have claimed an i_size + * extension and not yet dirtied it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; }; /* readdir cache (directory only) */ @@ -561,6 +618,8 @@ struct fuse_req { #ifdef CONFIG_FUSE_IO_URING void *ring_entry; void *ring_queue; + /** Defers fuse_request_end() to the ring task's task work */ + struct callback_head ring_end_work; #endif /** When (in jiffies) the request was created */ unsigned long create_time; @@ -716,17 +775,6 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; -/** - * DLM retry tracking for iomap write deadlock workaround. - * - * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. Tracks tasks that need to retry write operations due - * to DLM lock contention (-EAGAIN from FUSE server). - */ -struct fuse_dlm_retry { - bool retry_needed; -}; - /** * A Fuse connection. * @@ -1014,6 +1062,9 @@ struct fuse_conn { /* do we have support for dlm in the fs? */ unsigned int dlm:1; + /* Is extended lookup implemented by fs? */ + unsigned int lookupx:1; + /** Passthrough support for read/write IO */ unsigned int passthrough:1; @@ -1029,7 +1080,6 @@ struct fuse_conn { /* Use io_uring for communication */ unsigned int io_uring; - /* Does the filesystem support compound operations? */ unsigned int compound_open_getattr:1; /** Maximum stack depth for passthrough backing files */ @@ -1107,12 +1157,8 @@ struct fuse_conn { unsigned int req_timeout; } timeout; - /** - * XArray tracking tasks that need DLM retry. - * Maps task pointer -> struct fuse_dlm_retry. - * Temporary workaround for iomap write deadlock. - */ - struct xarray dlm_retry_tasks; + /* The foffset alignment in PAGE */ + unsigned int alignment_pages; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 431ae386872646..21af962106e1d7 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,7 +37,14 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); -bool __read_mostly enable_large_folios = true; +/* + * A folio wider than a block carries per-block dirty state, which + * iomap_writeback_folio() clears whole after ->writeback_range. A run + * fuse defers and puts back with folio_mark_dirty() comes back dirty over + * the whole folio, so the next pass sends blocks this client never wrote. + * Off until there is a way to restore only the blocks that were dirty. + */ +bool __read_mostly enable_large_folios; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); @@ -219,23 +226,6 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } - - /* - * Free the coherency gate here rather than in ->free_inode: that runs - * from an RCU callback, where percpu_free_rwsem() may sleep in - * rcu_sync_dtor() if the write side has not fully quiesced. No user - * can remain by eviction time: gate readers hold a file reference and - * a concurrent notify holds an inode reference. wb_inval_rwsem lives - * in the regular-file union arm and is only ever allocated for regular - * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked - * regular files still own a gate); a directory's overlapping - * readdir-cache fields must not be misread. - */ - if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { - percpu_free_rwsem(fi->wb_inval_rwsem); - kfree(fi->wb_inval_rwsem); - fi->wb_inval_rwsem = NULL; - } } static int fuse_reconfigure(struct fs_context *fsc) @@ -261,6 +251,150 @@ static ino_t fuse_squash_ino(u64 ino64) return ino; } +/* + * Handle statx-specific attribute updates with partial attribute support. + */ +static void fuse_change_attributes_common_sx(struct inode *inode, + struct fuse_attr *attr, + struct fuse_statx *sx, + u64 attr_valid, u32 cache_mask, + u64 evict_ctr) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + u32 returned_attrs = sx->mask & STATX_BASIC_STATS; + + lockdep_assert_held(&fi->lock); + + /* + * Clear returned basic stats from invalid mask. + * + * Don't do this if this is coming from a fuse_iget() call and there + * might have been a racing evict which would've invalidated the result + * if the attr_version would've been preserved. + * + * !evict_ctr -> this is create + * fi->attr_version != 0 -> this is not a new inode + * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request + */ + if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + set_mask_bits(&fi->inval_mask, returned_attrs, 0); + + fi->attr_version = atomic64_inc_return(&fc->attr_version); + + /* + * Only update i_time if we got all the attributes we care about. + * + * With writeback_cache (cache_mask set): cache_mask attributes are + * managed locally and their values from the server are ignored. + * So we only need all the OTHER attributes (non-cache_mask). + */ + if (cache_mask) { + /* writeback_cache: ignore cache_mask attrs, check everything else */ + if ((returned_attrs | cache_mask) == STATX_BASIC_STATS) + fi->i_time = attr_valid; + } else { + /* no writeback_cache: need all basic stats */ + if (returned_attrs == STATX_BASIC_STATS) + fi->i_time = attr_valid; + } + + /* + * Permission-check cache: independent of i_time so that a partial + * refresh which covers only mode/uid/gid (e.g. fuse_perm_getattr()) + * still extends the window during which fuse_permission() can hit + * the cache. Requires all three perm bits because generic_permission() + * needs the full triple. + */ + if ((returned_attrs & (STATX_MODE | STATX_UID | STATX_GID)) == + (STATX_MODE | STATX_UID | STATX_GID)) + fi->i_perm_time = attr_valid; + + /* + * Only update inode fields for attributes that were actually returned. + * TYPE is part of i_mode but already set during inode creation. + */ + if (returned_attrs & STATX_INO) + inode->i_ino = fuse_squash_ino(attr->ino); + if (returned_attrs & STATX_MODE) + inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); + if (returned_attrs & STATX_NLINK) + set_nlink(inode, attr->nlink); + if (returned_attrs & STATX_UID) + inode->i_uid = make_kuid(fc->user_ns, attr->uid); + if (returned_attrs & STATX_GID) + inode->i_gid = make_kgid(fc->user_ns, attr->gid); + if (returned_attrs & STATX_BLOCKS) + inode->i_blocks = attr->blocks; + + if (returned_attrs & STATX_ATIME) { + attr->atimensec = min_t(u32, attr->atimensec, NSEC_PER_SEC - 1); + inode_set_atime(inode, attr->atime, attr->atimensec); + } + /* mtime from server may be stale due to local buffered write */ + if ((returned_attrs & STATX_MTIME) && !(cache_mask & STATX_MTIME)) { + attr->mtimensec = min_t(u32, attr->mtimensec, NSEC_PER_SEC - 1); + inode_set_mtime(inode, attr->mtime, attr->mtimensec); + } + if ((returned_attrs & STATX_CTIME) && !(cache_mask & STATX_CTIME)) { + attr->ctimensec = min_t(u32, attr->ctimensec, NSEC_PER_SEC - 1); + inode_set_ctime(inode, attr->ctime, attr->ctimensec); + } + if (sx) { + /* Sanitize nsecs */ + sx->btime.tv_nsec = + min_t(u32, sx->btime.tv_nsec, NSEC_PER_SEC - 1); + + /* + * Btime has been queried, cache is valid (whether or not btime + * is available or not) so clear STATX_BTIME from inval_mask. + * + * Availability of the btime attribute is indicated in + * FUSE_I_BTIME + */ + set_mask_bits(&fi->inval_mask, STATX_BTIME, 0); + if (sx->mask & STATX_BTIME) { + set_bit(FUSE_I_BTIME, &fi->state); + fi->i_btime.tv_sec = sx->btime.tv_sec; + fi->i_btime.tv_nsec = sx->btime.tv_nsec; + } + } + + /* + * Common fields for both statx and getattr. + * + * inode->i_blkbits, which is what the page cache is tracked in, + * stays at the superblock block size; FUSE_INIT refuses a writeback + * connection whose block is not a page. What the server named per + * inode is reported as st_blksize out of fi->cached_i_blkbits. + */ + if (attr->blksize != 0) + fi->cached_i_blkbits = ilog2(attr->blksize); + else + fi->cached_i_blkbits = inode->i_sb->s_blocksize_bits; + + /* + * Don't set the sticky bit in i_mode, unless we want the VFS + * to check permissions. This prevents failures due to the + * check in may_delete(). + */ + fi->orig_i_mode = inode->i_mode; + if (!fc->default_permissions) + inode->i_mode &= ~S_ISVTX; + + fi->orig_ino = attr->ino; + + /* + * We are refreshing inode data and it is possible that another + * client set suid/sgid or security.capability xattr. So clear + * S_NOSEC. Ideally, we could have cleared it only if suid/sgid + * was set or if security.capability xattr was set. But we don't + * know if security.capability has been set or not. So clear it + * anyway. Its less efficient but should be safe. + */ + inode->i_flags &= ~S_NOSEC; +} + void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, struct fuse_statx *sx, u64 attr_valid, u32 cache_mask, @@ -271,6 +405,12 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, lockdep_assert_held(&fi->lock); + if (sx) { + return fuse_change_attributes_common_sx(inode, attr, sx, + attr_valid, cache_mask, + evict_ctr); + } + /* * Clear basic stats from invalid mask. * @@ -288,6 +428,7 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, fi->attr_version = atomic64_inc_return(&fc->attr_version); wake_up_all(&fc->attr_version_waitq); fi->i_time = attr_valid; + fi->i_perm_time = attr_valid; inode->i_ino = fuse_squash_ino(attr->ino); inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); @@ -382,14 +523,14 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is entirely under a write grant. + * Taking the server's answer would shrink i_size and have + * truncate_pagecache() throw the unwritten tail away. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, @@ -413,16 +554,38 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, !S_ISREG(inode->i_mode)) return cache_mask; - if (!fuse_dlm_write_grant_exists(fi)) - return cache_mask; - + /* + * A dirty mapping keeps the local attributes authoritative even + * when no grant is recorded: a fault dirties pages under a + * page-mkwrite lock that is never recorded, and a truncate revokes + * the tail grants itself while cached writes above the new size + * are still waiting for writeback. + */ if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; + /* + * The local size stays authoritative while the extension is + * covered by a write grant, and also while anything in + * [attr->size, size) is dirty or under writeback: those bytes + * exist only here, and taking the server's smaller size would + * truncate them away before they are ever sent. The grant check + * alone misses them, because a page-mkwrite grant is never + * recorded and a local truncate revokes its own tail grants. + * + * Both miss a write that has claimed its extension and not yet + * dirtied it: nothing is dirty there, and a NOTIFY can revoke the + * grant in between. With i_rwsem held shared several writers sit + * in that window at once, which is why they are counted rather + * than flagged. + */ if (have_size && size > (loff_t) attr->size && - fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE)) + (atomic_read(&fi->size_extenders) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE) || + filemap_range_needs_writeback(inode->i_mapping, attr->size, + size - 1))) cache_mask |= STATX_SIZE; return cache_mask; @@ -438,13 +601,10 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr loff_t oldsize; struct timespec64 old_mtime; bool have_size = !sx || (sx->mask & STATX_SIZE); - u64 srv_size; cache_mask = fuse_attr_cache_mask(inode, attr, have_size); spin_lock(&fi->lock); - srv_size = attr->size; - if (cache_mask & STATX_SIZE) attr->size = i_size_read(inode); @@ -463,19 +623,6 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr return; } - /* - * srv_size is the size the server reported before the writeback - * cache_mask above replaced attr->size with the local value. It - * bounds how far the server can hold data, letting the iomap write - * path zero-fill expansion read-modify-writes instead of sending - * READ requests, see fuse_iomap_read_folio_range(). Only ever grow - * it here: stale attributes were rejected above and truncation - * lowers it directly. - */ - if (have_size && S_ISREG(inode->i_mode) && - (loff_t) srv_size > fi->server_size) - fi->server_size = srv_size; - old_mtime = inode_get_mtime(inode); fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, evict_ctr); @@ -485,8 +632,9 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr * In case of writeback_cache enabled, the cached writes beyond EOF * extend local i_size without keeping userspace server in sync. So, * attr->size coming from server can be stale. We cannot trust it. + * Only update i_size if SIZE was actually returned by the server. */ - if (!(cache_mask & STATX_SIZE)) + if (have_size && !(cache_mask & STATX_SIZE)) i_size_write(inode, attr->size); spin_unlock(&fi->lock); @@ -502,12 +650,13 @@ static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr */ if (!(cache_mask & STATX_SIZE) && S_ISREG(inode->i_mode)) { bool inval = false; + bool have_mtime = !sx || (sx->mask & STATX_MTIME); - if (oldsize != attr->size) { + if (have_size && oldsize != attr->size) { truncate_pagecache(inode, attr->size); if (!fc->explicit_inval_data) inval = true; - } else if (fc->auto_inval_data) { + } else if (have_mtime && fc->auto_inval_data) { struct timespec64 new_mtime = { .tv_sec = attr->mtime, .tv_nsec = attr->mtimensec, @@ -747,6 +896,9 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* Address only; see fuse_notify_ctx_enter() */ +const char fuse_notify_ctx_key[1]; + /* * Revoke the DLM grants backing an invalidated byte range. Grants are * recorded page-aligned, so widen the revoke to page boundaries: dropping @@ -770,44 +922,63 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * Drop a page-cache range on behalf of a NOTIFY invalidate. * * invalidate_inode_pages2_range() waits out folios under writeback and - * launders dirty ones, both of which need a FUSE_WRITE reply. While - * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, - * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() - * parks the request on fi->queued_writes until fuse_release_nowrite(). A - * server that revokes from inside the handler it is revoking for then - * deadlocks against its own reply. fuse_do_setattr() states the same rule - * for its own invalidate. + * launders dirty ones, both of which need a FUSE_WRITE reply. It is only + * needed when the range can hold data the server has not seen. + * + * @may_be_dirty false says it cannot, on the strength of the DLM range + * record: every way a folio gets dirtied under a grant raises that record + * before the data lands, so a range it reports clean has no dirty folio to + * launder. invalidate_mapping_pages() then drops the same folios without + * ever waiting for the server. + * + * The same substitution is forced while writepages are frozen + * (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, pre-SETATTR flush), + * where no reply can arrive because fuse_flush_writepages() parks the + * request on fi->queued_writes until fuse_release_nowrite(). A server that + * revokes from inside the handler it is revoking for would otherwise + * deadlock against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. There the dirty folios are left behind, and the + * freezes that span a request drop the cache themselves once they complete: + * fuse_do_setattr() invalidates the mapping after releasing the freeze, the + * O_TRUNC open path calls truncate_pagecache(). * - * So while frozen use invalidate_mapping_pages(), which skips dirty and - * under-writeback folios and never blocks. The stale clean folios still - * go, and the freezes that span a request drop the cache themselves once - * they complete: fuse_do_setattr() invalidates the mapping after releasing - * the freeze, the O_TRUNC open path calls truncate_pagecache(). + * A drop that can launder writes the range back itself first. + * fuse_launder_folio() is handed one folio at a time and sends a FUSE_WRITE + * for each, where a writeback pass batches the same bytes up to + * fc->max_write; what it sends the drop then only waits for. */ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); + loff_t last; bool frozen; spin_lock(&fi->lock); frozen = fi->writectr < 0; spin_unlock(&fi->lock); - if (frozen) + if (frozen || !may_be_dirty) { invalidate_mapping_pages(inode->i_mapping, start, end); - else - invalidate_inode_pages2_range(inode->i_mapping, start, end); + return; + } + + last = end == (pgoff_t)-1 ? LLONG_MAX : + (((loff_t)end + 1) << PAGE_SHIFT) - 1; + filemap_write_and_wait_range(inode->i_mapping, + (loff_t)start << PAGE_SHIFT, last); + invalidate_inode_pages2_range(inode->i_mapping, start, end); } int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; + bool tracked; inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) @@ -834,63 +1005,102 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, forget_all_cached_acls(inode); security_inode_invalidate_secctx(inode); if (offset >= 0) { + /* + * Everything below drives this inode's page cache on behalf + * of the revoke, so mark the task: writeback reached from + * here must not send a DLM request. See + * fuse_iomap_writeback_range(). + */ + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; + pg_start = offset >> PAGE_SHIFT; if (len <= 0) pg_end = -1; else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + /* - * A data invalidation means another (remote) entity is modifying - * the file. Two things happen here: + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. * - * 1. Coherency. Drop the affected page-cache range so no local - * read returns a folio the remote modify has superseded. This - * runs under the write side of the per-inode coherency gate - * (wb_inval_rwsem), which fences cache-serving buffered reads - * and buffered writes out for the whole invalidate. Unlike the - * old best-effort trylock this BLOCKS -- the notify has - * priority: percpu_down_write() parks new gate readers, drains - * in-flight ones, then invalidates. A blocking writer here is - * safe only under a server that services request replies on - * threads other than the one delivering this notify: the write - * side waits for gate readers to drain, and a cache-miss read - * holds the read side across its FUSE_READ round-trip. redfs' - * dlm server provides that contract; a server that cannot must - * not enable writeback+dlm. + * A fenced writer may be waiting for a FUSE_READ or a + * FUSE_WRITE reply, so this waits on the server the same way + * the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. + */ + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); + + /* + * A data invalidation means another (remote) entity is + * modifying the file. Two things happen here: * - * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under - * fi->lock, updated for every data invalidation) of how fast - * these arrive; when they come in a rapid stream -- a remote - * writer repeatedly invalidating -- and the inode is also open - * for writing here, latch it into direct IO until the last - * writer closes or it is mmapped. When latched, drop the whole - * mapping rather than just the notified range, or dirty folios - * outside it would be invisible to the forced direct reads - * (stale read / lost write). Latching is opt-in via the - * enable_notify_dio module parameter and off by default; the - * average is kept up to date either way, so enabling it at - * runtime takes effect on the next storm rather than after a - * warm-up. Clearing it at runtime stops new latches but lets + * 1. Coherency. Drop the affected page-cache range so no + * local read returns a folio the remote modify has + * superseded. Nothing is fenced out for it. A read + * racing the drop either misses and refetches or returns + * data that was current when it was copied. A write + * racing it is caught on the way out instead: its bytes + * were recorded before they were dirtied, this revoke + * marks the range rather than forgetting it, and + * writeback holds the range again before sending + * anything it finds marked that way. + * + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), + * under fi->lock, updated for every data invalidation) of + * how fast these arrive; when they come in a rapid stream + * -- a remote writer repeatedly invalidating -- and the + * inode is also open for writing here, latch it into + * direct IO until the last writer closes or it is mmapped. + * When latched, drop the whole mapping rather than just + * the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost + * write). Latching is opt-in via the enable_notify_dio + * module parameter and off by default; the average is kept + * up to date either way, so enabling it at runtime takes + * effect on the next storm rather than after a warm-up. + * Clearing it at runtime stops new latches but lets * already-latched inodes run out on the usual exits (last * writer closes, or mmap). * - * The gate (and the average) exist only for writeback+dlm regular - * files; elsewhere wb_sem is NULL and the invalidate runs - * unserialized (best-effort), as before. An mmapped inode - * keeps the gate -- fuse_cache_read_iter() and - * fuse_cache_write_iter() enter it unconditionally and rely - * on the revoke staying fenced -- but is never latched: - * a mapping needs the page cache, and fuse_file_mmap() - * reverts any latch it races with. + * The average and the latch exist only for writeback+dlm + * regular files; elsewhere there is no record to consult and + * the range is dropped as it always was. An mmapped inode is + * never latched: a mapping needs the page cache, and + * fuse_file_mmap() reverts any latch it races with. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi)) - wb_sem = fi->wb_inval_rwsem; + tracked = S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi); - if (wb_sem) { + if (tracked) { bool hot, has_writer, latched = false; + bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); @@ -898,22 +1108,47 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, spin_unlock(&fi->lock); /* - * Priority write side: park new gate readers, - * drain in-flight ones, then invalidate. Blocks - * (unlike the old trylock) -- see the contract in - * the comment above. + * What this notify has to do. Nothing cached in the + * range means the drop is a no-op and the revoke is + * the whole job. Otherwise the page cache says + * whether the drop has to launder, which is what + * makes it wait for a FUSE_WRITE reply. */ - percpu_down_write(wb_sem); + has_pages = filemap_range_has_page(inode->i_mapping, + offset, end_byte); + may_be_dirty = filemap_range_needs_writeback( + inode->i_mapping, offset, end_byte); /* - * Revoke the DLM lock range under the gate write - * side, atomically with the page drop: gate readers - * re-validate their grant right after entering, and - * a grant that passed that check must stay visible - * for their whole gate hold. + * Put unwritten data on the server while the grant + * still covers it, rather than leaving it to the drop + * below. After the revoke writeback would have to + * take the range again to send those bytes: a DLM + * round trip from inside the handler the server is + * waiting on. do_writepages() runs in this context, + * so the grant is asked for before the revoke. + * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * + * Waited out here rather than left to the drop, which + * launders when the record says the range may be dirty + * and so waits for these same replies. One explicit + * wait, before the revoke, and the drop then finds + * nothing under writeback to block on. Either way a + * server that revokes from a thread it also needs to + * answer FUSE_WRITE on deadlocks here, the same + * contract fuse_notify_invalidate_range() states for a + * frozen inode. The error is left to the mapping, + * where fsync collects it. */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_revoke_inval_range(fi, offset, len); + if (has_pages && may_be_dirty) + filemap_write_and_wait_range(inode->i_mapping, + offset, end_byte); + + fuse_dlm_revoke_inval_range(fi, offset, len); if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && @@ -929,28 +1164,35 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads). Otherwise just the - * notified range. + * the forced direct reads), and the record says + * nothing about the rest of the file, so launder. + * Otherwise just the notified range, and only if + * anything is cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1); - else + fuse_notify_invalidate_range(inode, 0, -1, true); + else if (has_pages) fuse_notify_invalidate_range(inode, pg_start, - pg_end); - - percpu_up_write(wb_sem); + pg_end, + may_be_dirty); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (DAX, backing, non-regular, - * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. */ + /* + * No record on this inode (DAX, backing, non-regular, + * or no DLM), so assume the range can hold unwritten + * data and drop it as before. + */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end, + true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); + fuse_notify_ctx_leave(notify_ctx); } iput(inode); return 0; @@ -1364,11 +1606,11 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, fc->initialized = 0; fc->connected = 1; fc->dlm = 1; + fc->lookupx = 1; /* module option for now */ fc->compound_open_getattr = enable_compound; - xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1421,7 +1663,6 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); - xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } EXPORT_SYMBOL_GPL(fuse_conn_put); @@ -1751,8 +1992,31 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, } if (flags & FUSE_ASYNC_DIO) fc->async_dio = 1; - if (flags & FUSE_WRITEBACK_CACHE) + if (flags & FUSE_WRITEBACK_CACHE) { + /* + * A buffered write goes through iomap, which + * tracks a folio a block at a time and fills + * any block the write covers only part of. + * Writeback then sends whole dirty blocks. + * Both of those are cut at the page in this + * filesystem: fuse_dlm_buffered_write() sends + * the unaligned edges of a write to the server + * itself so that no partly written block is + * ever dirtied, and it cuts at PAGE_SIZE. + * + * A block that is not a page breaks that, and + * quietly: the fill would come back for the + * remainder of an edge block, from a server + * that need not hold anything there. Refuse + * the connection instead. + */ + if (fm->sb->s_blocksize_bits != PAGE_SHIFT) { + pr_err("fuse: writeback cache needs a page sized block, got %lu\n", + fm->sb->s_blocksize); + ok = false; + } fc->writeback_cache = 1; + } if (flags & FUSE_PARALLEL_DIROPS) fc->parallel_dirops = 1; if (flags & FUSE_HANDLE_KILLPRIV) @@ -1822,6 +2086,14 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->max_stack_depth = arg->max_stack_depth; fm->sb->s_stack_depth = arg->max_stack_depth; } + + if (flags & FUSE_ALIGN_PG_ORDER) { + if (arg->align_page_order > 0) { + fc->alignment_pages = + (1UL << arg->align_page_order) + >> PAGE_SHIFT; + } + } if (flags & FUSE_NO_EXPORT_SUPPORT) fm->sb->s_export_op = &fuse_export_fid_operations; if (flags & FUSE_ALLOW_IDMAP) { diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index ebe2735c186e08..4c4ecf6cb4ee89 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -461,6 +461,8 @@ struct fuse_file_lock { * init_out.request_timeout contains the timeout (in secs) * FUSE_INVAL_INODE_ENTRY: invalidate inode aliases when doing inode invalidation * FUSE_EXPIRE_INODE_ENTRY: expire inode aliases when doing inode invalidation + * FUSE_ALIGN_PG_ORDER: page order (power of 2 exponent for number of pages) for + * optimal io-size alignment * FUSE_URING_REDUCED_Q: Client (kernel) supports less queues - Server is free * to register between 1 and nr-core io-uring queues * FUSE_SETATTR_WRITEBACK: kernel marks writeback-initiated SETATTR requests @@ -610,6 +612,14 @@ struct fuse_file_lock { */ #define FUSE_OPEN_KILL_SUIDGID (1 << 0) +/** + * Lookup flags + * FUSE_LOOKUPX_FOR_REVALIDATE: lookup called from revalidate + * FUSE_LOOKUPX_TARGET_WASDIR: (hint) the lookup target was a directory + */ +#define FUSE_LOOKUPX_FOR_REVALIDATE (1 << 0) +#define FUSE_LOOKUPX_TARGET_WAS_DIR (1 << 1) + /** * setxattr flags * FUSE_SETXATTR_ACL_KILL_SGID: Clear SGID when system.posix_acl_access is set @@ -696,6 +706,9 @@ enum fuse_opcode { */ FUSE_COMPOUND = 101, + /* Extented lookup operation */ + FUSE_LOOKUPX = 102, + /* CUSE specific operations */ CUSE_INIT = 4096, @@ -732,6 +745,15 @@ struct fuse_entry_out { struct fuse_attr attr; }; +struct fuse_lookupx_in { + uint32_t lookup_flags; +}; + +struct fuse_lookupx_out { + struct fuse_entry_out entry; + uint32_t mask; /* Mask of valid attributes in statx format */ +}; + struct fuse_forget_in { uint64_t nlookup; }; @@ -943,6 +965,9 @@ struct fuse_init_in { #define FUSE_COMPAT_INIT_OUT_SIZE 8 #define FUSE_COMPAT_22_INIT_OUT_SIZE 24 +/* + * align_page_order: Number of pages for optimal IO, or a multiple of that + */ struct fuse_init_out { uint32_t major; uint32_t minor; @@ -957,7 +982,9 @@ struct fuse_init_out { uint32_t flags2; uint32_t max_stack_depth; uint16_t request_timeout; - uint16_t unused[11]; + uint8_t align_page_order; + uint8_t padding; + uint16_t unused[10]; }; #define CUSE_INIT_INFO_MAX 4096