diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 5f426228f4c76a..97f7cdcab43289 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -726,6 +726,10 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, memset(&inarg, 0, sizeof(inarg)); memset(&outentry, 0, sizeof(outentry)); inarg.flags = flags; + + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; inarg.mode = mode; inarg.umask = current_umask(); @@ -2106,34 +2110,24 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. * - * Revoke and drop under the coherency gate write side, - * like the NOTIFY invalidate path: a gate reader that - * already re-validated its grant must not have the - * lock tree and the cache yanked mid-hold, or it - * would repopulate the truncated range trusting a - * grant that no longer exists. Waiting for gate - * readers here is safe: we hold i_rwsem exclusive, so - * no gate holder can be waiting on it (the write path - * takes i_rwsem before the gate, the read path never - * takes it). + * Dropping every grant here does not need a reader or + * writer fenced out: truncate_pagecache() discards the + * folios rather than writing them, and a write racing + * this is a write racing an O_TRUNC open, which has no + * order to preserve. i_rwsem is held exclusive + * anyway, so no cached write is in progress. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); - if (wb_sem) - percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2237,23 +2231,17 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* - * Revoke and drop under the coherency gate write side; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out gate - * readers cannot deadlock. + * Revoke past the new size and drop what is beyond it; see + * the atomic-O_TRUNC branch above for why this needs nothing + * fenced out. i_rwsem is held exclusive here as well. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, + U64_MAX); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - if (wb_sem) - percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7c5b5e8b4ba268..dd5aa9cb7ce446 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -71,6 +71,17 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; + /* + * With the writeback cache the kernel owns append positioning: + * writeback sends FUSE_WRITE with explicit offsets, and a server + * that opens its backing file O_APPEND has pwrite(2) ignore them + * (Linux appends regardless of offset). Any re-sent or reordered + * run is then placed at EOF: duplicated data and a growing file. + * Do not hand the flag to the server at all. + */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -174,6 +185,10 @@ static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) open_in.flags &= ~O_TRUNC; + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + open_in.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -384,10 +399,26 @@ static int fuse_open(struct inode *inode, struct file *file) if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); if (!err) { - if (is_truncate) + if (is_truncate) { + /* + * Every grant goes with the cache, as on the + * fuse_do_setattr() O_TRUNC path: a record left + * behind would keep naming bytes the folios no + * longer hold. i_rwsem is held exclusive + * (is_wb_truncate), so no cached write is mid-record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { + /* + * Write back first: the drop launders whatever it + * finds dirty a folio at a time, a FUSE_WRITE per + * page, where writeback batches the same bytes. + */ + filemap_write_and_wait(inode->i_mapping); invalidate_inode_pages2(inode->i_mapping); + } } if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); @@ -469,8 +500,8 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * If this release dropped the last writer, fuse_prepare_release() * cleared the forced-direct-IO latch (under fi->lock). Drop any clean * folios a read racing the latch may have repopulated so they cannot be - * served stale once caching mode resumes. No inode lock or - * wb_inval_rwsem: release may run on the fuse server thread (async fput + * served stale once caching mode resumes. No inode lock: release may + * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could * stall the connection. Writes were routed direct while latched, so * only clean folios exist and this invalidate is server-free; the last @@ -661,6 +692,16 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, if (fuse_is_bad(inode)) return -EIO; + /* + * Get the sending out of the way before the lock. It is the long + * part: a pass takes back every grant it finds gone, a cluster round + * trip each, and cached writers hold i_rwsem shared + * (fuse_cache_wr_exclusive_lock()) and would all wait behind it. + * Errors are left to the pass below, which collects them from the + * mapping. + */ + filemap_fdatawrite_range(file->f_mapping, start, end); + inode_lock(inode); /* @@ -1007,16 +1048,137 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, return 0; } +/** + * fuse_read_folio_range - read part of a folio from the server + * @file: file to read through + * @folio: the folio to fill + * @off: offset within @folio to start at + * @len: bytes to read + * + * fuse_do_readfolio() cannot serve a partial folio: it asks for + * page_zeroing, and fuse_copy_folio() answers that by zeroing the whole + * folio whenever the request covers less than all of it. Ask without it + * and zero exactly what the reply left short, which is the server saying + * the file ends there. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_range(struct file *file, struct folio *folio, + size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = folio_pos(folio) + off; + struct fuse_folio_desc desc = { + .offset = off, + .length = len, + }; + struct fuse_io_args ia = { + .ap.args.out_pages = true, + .ap.num_folios = 1, + .ap.folios = &folio, + .ap.descs = &desc, + }; + ssize_t res; + + /* Don't overflow end offset */ + if (pos + (desc.length - 1) == LLONG_MAX) + desc.length--; + + fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) { + /* See fuse_do_readfolio() for why READ can return -EDEADLK */ + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) + res = AOP_TRUNCATED_PAGE; + return res; + } + + if (res < desc.length) + folio_zero_range(folio, off + res, desc.length - res); + + return 0; +} + +/** + * fuse_read_folio_merge - fill @folio without disturbing what it holds + * @file: file to read through + * @folio: the folio to fill + * + * iomap tracks a folio a block at a time, and a write that covered some + * of its blocks and not others leaves it valid in the ones it covered + * and not in the rest. The valid ones hold what that write put there, + * which writeback may not have sent yet; reading over them would lose + * it. Fetch the rest, in as few requests as the gaps allow. + * + * A folio the page cache tracks in one piece has no per block state to + * ask, and none to have: it is dirty only if a write covered it whole, + * and then it is valid and never reaches here. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_merge(struct file *file, struct folio *folio) +{ + size_t bsize = i_blocksize(folio->mapping->host); + size_t size = folio_size(folio); + size_t off = 0; + + while (off < size) { + size_t run = 0; + int err; + + /* Skip what the folio already holds */ + while (off < size && + iomap_is_partially_uptodate(folio, off, bsize)) + off += bsize; + + /* Take the gap behind it in one request */ + while (off + run < size && + !iomap_is_partially_uptodate(folio, off + run, bsize)) + run += bsize; + + if (!run) + break; + + err = fuse_read_folio_range(file, folio, off, run); + if (err) + return err; + off += run; + } + + return 0; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); int err; err = -EIO; if (fuse_is_bad(inode)) goto out; - err = fuse_do_readfolio(file, folio, 0, folio_size(folio)); + /* + * Writeback unlocks a folio as soon as it has handed it over, with + * the writeback flag still on it, so this can be reached while a + * FUSE_WRITE is still reading out of it. Filling it now would + * rewrite what is being sent, and past the end of the file the reply + * comes back short and zeroes it. Nothing reached here before a + * partial write started leaving folios invalid, because a dirty + * folio was always valid and never came this way. + */ + folio_wait_writeback(folio); + + /* + * Only a folio still holding what a write put in it has anything to + * keep. One that was merely being written back is clean by now, + * waited out just above, and reads whole. + */ + if (fc->writeback_cache && folio_test_dirty(folio)) + err = fuse_read_folio_merge(file, folio); + else + err = fuse_do_readfolio(file, folio, 0, folio_size(folio)); if (!err) folio_mark_uptodate(folio); @@ -1026,17 +1188,33 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return err; } +/* + * What iomap_file_buffered_write() carries for fuse, reached from the + * read-back callback as iter->private. @file is what that callback needs + * anyway, so the retry flag rides along and lives exactly as long as the + * call, with nothing to allocate or free. + */ +struct fuse_iomap_write_ctx { + struct file *file; + /* fuse_iomap_read_folio_range() hit AOP_TRUNCATED_PAGE */ + bool retry_needed; + /* + * The grant ->get_folio pinned, live until ->put_folio drops it. + * Here rather than on their stacks, which do not span the pair. + */ + struct fuse_dlm_span pin; +}; + static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct folio *folio, loff_t pos, size_t len) { - struct file *file = iter->private; - struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_iomap_write_ctx *ctx = iter->private; + struct file *file = ctx->file; size_t off = offset_in_folio(folio, pos); int ret; - ret = fuse_do_readfolio(file, folio, off, len); + ret = fuse_read_folio_range(file, folio, off, len); /* * TEMPORARY WORKAROUND for iomap write deadlock: @@ -1047,7 +1225,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * * However, iomap doesn't understand AOP_TRUNCATED_PAGE. * We need to: - * 1. Mark the retry flag (caller stored it in xarray) + * 1. Mark the retry flag on the caller's write context * 2. Convert to -EAGAIN so iomap sees an error * 3. Let fuse_cache_write_iter() detect and retry * @@ -1058,13 +1236,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. */ if (ret == AOP_TRUNCATED_PAGE) { - struct fuse_dlm_retry *retry; - unsigned long task_key = (unsigned long)current; - - retry = xa_load(&fc->dlm_retry_tasks, task_key); - if (retry) { - retry->retry_needed = true; - } + ctx->retry_needed = true; /* Convert to -EAGAIN for iomap */ ret = -EAGAIN; @@ -1154,6 +1326,46 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * Readahead fills the page cache past the range the reader locked, + * so take a DLM read grant over the whole window here too. Folios + * the server handed out no lock for are folios it will not revoke + * when a remote node writes them, and a later read would be served + * from stale cache. Take the grant before any folio is pulled off + * @rac, so the window is either fully covered or not populated. + * + * Speculative pages are not worth serving uncovered: on a failed + * request drop the window and let read_pages() clean up the folios + * left in @rac. A server without DLM support answers -ENOSYS and + * clears fc->dlm, which is not a failure. + * + * ->readahead is entered with every folio of the window already + * locked, and pulling one off @rac is what unlocks it, so the round + * trip is taken under those locks. See the readahead line of + * Documentation/filesystems/locking.rst. + * + * What keeps that from closing a cycle is the direction a revoke + * travels. This asks for a read grant on a range it holds nothing + * on, so the lock in the way is another node's, and the revoke that + * frees it is sent there. Nothing here has to run for this request + * to be answered, so the folios stay locked only for as long as the + * round trip. + * + * A window this already holds part of is the open edge: the query + * fails for the whole of it, so the request goes out while that part + * is still held, and a revoke for that part is sent here. Whether + * the server can answer while a revoke of its own is outstanding is + * not something this side can know. + */ + if (fc->writeback_cache && fc->dlm) { + int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), + readahead_length(rac), + FUSE_PAGE_LOCK_READ); + + if (err < 0 && err != -ENOSYS) + return; + } + max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); @@ -1227,21 +1439,12 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); -/* - * Bound on re-requesting a revoked DLM grant before a cached read is - * served unlocked; see fuse_cache_read_iter(). - */ -#define FUSE_DLM_READ_RETRIES 3 - static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; - int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1259,65 +1462,35 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* - * Fence the cache-serving read against a NOTIFY invalidate so we never - * hand back a folio the server has just superseded. The gate read side - * is per-CPU cheap; the NOTIFY holds the write side with priority. - * Re-check the forced-DIO latch under it: if a storm latched us while we - * waited on a pending writer, reroute to direct like the buffered write - * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * A NOTIFY invalidate racing this read drops the folios it + * supersedes, so the read either misses and refetches or returns + * data that was current when it was copied. There is nothing to + * fence: unlike a write, a read leaves nothing behind that could + * reach the server under a grant it no longer holds. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; + if (fuse_inode_force_dio(inode)) { + size_t count = iov_iter_count(to); -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); - } /* - * The DLM lock was requested before entering the gate, and - * the NOTIFY invalidate we may just have waited on revokes - * locks under the gate write side. Re-check the grant here - * and re-request with the gate dropped, so a - * FUSE_DLM_WB_LOCK round trip never parks a pending - * invalidate behind our own gate hold. Once the check - * passes the lock cannot go away for the rest of the gate - * hold. A failed or unrecorded request falls through - * unlocked, as before: the retry is taken even then (the - * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would - * be re-requested forever. The retry is also bounded: a - * remote writer can revoke each successful grant before - * the gate is re-entered, and a reader-only inode has no - * force-DIO latch to end such a storm, so after - * FUSE_DLM_READ_RETRIES re-requests the read is served - * unlocked rather than looping without bound. + * A write that passed this same check just before the latch + * took hold dirtied the page cache after the notify dropped + * it, and a direct read does not look there. Send it first. */ - if (!lock_err && fc->dlm && tries-- > 0 && - !fuse_dlm_lock_is_held(fi, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ)) { - percpu_up_read(wb_sem); - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - goto retry; + if (count) { + res = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + if (res) + return res; } + return fuse_direct_read_iter(iocb, to); } res = generic_file_read_iter(iocb, to); - if (wb_sem) - percpu_up_read(wb_sem); - return res; } @@ -1458,7 +1631,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, { struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); - unsigned offset = pos & (PAGE_SIZE - 1); size_t count = 0; unsigned int num; int err = 0; @@ -1485,7 +1657,13 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, if (mapping_writably_mapped(mapping)) flush_dcache_folio(folio); - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; + /* + * From @pos, not carried across iterations: a write landing + * inside a folio it does not start covers the rest of it, and + * a residue kept from that lands the next folio's copy past + * its end. + */ + folio_offset = offset_in_folio(folio, pos); bytes = min(folio_size(folio) - folio_offset, num); tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); @@ -1515,9 +1693,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, count += tmp; pos += tmp; num -= tmp; - offset += tmp; - if (offset == folio_size(folio)) - offset = 0; /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) @@ -1529,7 +1704,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, ia->write.folio_locked = true; break; } - if (!fc->big_writes || offset != 0) + /* + * Carry on only from a folio boundary: a copy that stopped + * short leaves the next one starting inside a folio, which is + * one request's worth on its own. + */ + if (!fc->big_writes || folio_offset + tmp != folio_size(folio)) break; } @@ -1685,10 +1865,105 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) } } +/* + * How many times a writer confirms its grant again before giving up on + * the range. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation. + */ +#define FUSE_DLM_PIN_RETRIES 16 + +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + unsigned int tries = FUSE_DLM_PIN_RETRIES; + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (!tries--) + return -EIO; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + +/* + * The grant over the bytes about to be copied, held until ->put_folio + * hands the folio back dirty. Nothing is locked yet, which is what lets + * this wait out a revoke and ask for the grant again; ->writeback_range + * can do neither. + */ +static struct folio *fuse_iomap_get_folio(struct iomap_iter *iter, loff_t pos, + unsigned int len) +{ + struct fuse_iomap_write_ctx *ctx = iter->private; + struct folio *folio; + int err; + + err = fuse_dlm_pin_write(ctx->file, &ctx->pin, pos, len); + if (err) + return ERR_PTR(err); + + folio = iomap_get_folio(iter, pos, len); + if (IS_ERR(folio)) + fuse_dlm_unpin(get_fuse_inode(iter->inode)); + + return folio; +} + +/* iomap_write_end() has dirtied whatever was copied by now */ +static void fuse_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int copied, struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); + fuse_dlm_unpin(get_fuse_inode(inode)); +} + static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; +static const struct iomap_write_ops fuse_iomap_dlm_write_ops = { + .get_folio = fuse_iomap_get_folio, + .put_folio = fuse_iomap_put_folio, + .read_folio_range = fuse_iomap_read_folio_range, +}; + static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -1707,30 +1982,16 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, struct iov_iter *from, struct file *file) { - struct fuse_conn *fc = get_fuse_conn(file_inode(file)); - ssize_t written, total_written = 0; - /* * TEMPORARY WORKAROUND for iomap write deadlock: * - * Stack-allocate retry state and register it before calling - * iomap. If fuse_iomap_read_folio_range() encounters - * AOP_TRUNCATED_PAGE, it will mark retry_needed. - * - * Stack allocation ensures no memory leaks - the state is - * valid for the duration of this function call and is - * automatically cleaned up. + * The context fuse_iomap_read_folio_range() marks when it hits + * AOP_TRUNCATED_PAGE. iomap hands it back through its private + * pointer, which fuse needs for @file either way. */ - struct fuse_dlm_retry retry_state = { - .retry_needed = false, - }; - unsigned long task_key = (unsigned long)current; - int xa_ret; - - xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, - &retry_state, GFP_KERNEL)); - if (xa_ret) - return xa_ret; + struct fuse_iomap_write_ctx ctx = { .file = file }; + struct fuse_conn *fc = get_fuse_conn(file_inode(file)); + ssize_t written, total_written = 0; retry: /* @@ -1741,14 +2002,16 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * the next iteration with iov_iter already drained, and iomap * would re-enter with len==0 and livelock on a 0-length mapping. */ - retry_state.retry_needed = false; + ctx.retry_needed = false; /* * Use iomap so that we can do granular uptodate reads * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, file); + fc->dlm ? &fuse_iomap_dlm_write_ops : + &fuse_iomap_write_ops, + &ctx); if (written > 0) total_written += written; @@ -1760,17 +2023,12 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * The folio has been unlocked by fuse_do_readfolio(), * breaking the ABBA deadlock with page invalidation. * - * Keep the entry in xarray and reuse it for the retry. - * * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE * retry support. */ - if (retry_state.retry_needed && iov_iter_count(from)) + if (ctx.retry_needed && iov_iter_count(from)) goto retry; - /* Remove from xarray now that we're done */ - xa_erase(&fc->dlm_retry_tasks, task_key); - return written < 0 ? written : total_written; } @@ -1786,6 +2044,7 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, struct file *file, size_t len, bool through) { + struct fuse_dlm_span pin; size_t hidden; ssize_t res; @@ -1795,8 +2054,21 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, /* Cap the iterator to this chunk, keeping the tail for later chunks. */ hidden = iov_iter_count(from) - len; iov_iter_truncate(from, len); - res = through ? fuse_perform_write(iocb, from, true) - : fuse_writeback_write_iter(iocb, from, file); + if (through) { + /* + * These bytes never enter the page cache, so a revoke cannot + * find them by flushing it. Hold the grant across the + * FUSE_WRITE instead, which is the reply the revoke already + * waits for when the same bytes go through writeback. + */ + res = fuse_dlm_pin_write(file, &pin, iocb->ki_pos, len); + if (!res) { + res = fuse_perform_write(iocb, from, true); + fuse_dlm_unpin(get_fuse_inode(file_inode(file))); + } + } else { + res = fuse_writeback_write_iter(iocb, from, file); + } /* Restore from the iterator's own residue, so short writes/errors * (which leave it partly advanced) reexpand to the exact remainder. */ iov_iter_reexpand(from, iov_iter_count(from) + hidden); @@ -1805,18 +2077,25 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, } /* - * Buffered write under DLM. A partial page dirtied for writeback would - * have to be completed by reading the untouched remainder back from the - * server, and for a write past the server EOF that READ can only return - * zero bytes: a wasted round trip per unaligned edge. So cache only the - * page-aligned interior, whole pages need no read-modify-write, and - * route the unaligned head and tail straight through to the server. The - * writethrough path writes just those bytes and leaves the page - * non-uptodate, doing no read, and each edge lands as an independent - * FUSE_WRITE carrying FUSE_WRITE_CACHE like the writeback it replaces, - * so writers sharing a boundary page accumulate their bytes on the - * server. Aligned writes take the interior path whole; a - * sub-page write with no aligned interior goes fully through. + * Buffered write under DLM. A partly written page dirtied for + * writeback would have to be completed by reading the untouched + * remainder back from the server, and for a write past the server EOF + * that READ can only return zero bytes: a wasted round trip per + * unaligned edge. So cache only the page-aligned interior, whole pages + * need no read-modify-write, and route the unaligned head and tail + * straight through to the server. The writethrough path writes just + * those bytes and leaves the page non-uptodate, doing no read, and each + * edge lands as an independent FUSE_WRITE carrying FUSE_WRITE_CACHE + * like the writeback it replaces, so writers sharing a boundary page + * accumulate their bytes on the server. Aligned writes take the + * interior path whole; a sub-page write with no aligned interior goes + * fully through. + * + * The page is the block here: iomap tracks a folio a block at a time + * and __iomap_write_begin() skips the fill only for a block the write + * covers whole, so the cut has to land on block bounds. A writeback + * connection is refused unless the two are the same size; see + * process_init_reply(). */ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, struct iov_iter *from, @@ -1826,32 +2105,38 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, loff_t end = pos + iov_iter_count(from); loff_t mid_start = round_up(pos, PAGE_SIZE); loff_t mid_end = round_down(end, PAGE_SIZE); + /* Unaligned head, cached interior of whole pages, unaligned tail */ + const struct { + loff_t len; + bool through; + } chunk[] = { + { mid_start - pos, true }, + { mid_end - mid_start, false }, + { end - mid_end, true }, + }; ssize_t res, total = 0; + unsigned int i; /* No whole page inside the write: nothing cacheable, all through. */ if (mid_end <= mid_start) - return fuse_perform_write(iocb, from, true); + return fuse_dlm_write_chunk(iocb, from, file, + iov_iter_count(from), true); - /* Unaligned head [pos, mid_start): through. */ - res = fuse_dlm_write_chunk(iocb, from, file, mid_start - pos, true); - if (res < 0) - return res; - total += res; - if (res < mid_start - pos) - return total; - - /* Aligned interior [mid_start, mid_end): cached whole pages. */ - res = fuse_dlm_write_chunk(iocb, from, file, mid_end - mid_start, false); - if (res < 0) - return total; - total += res; - if (res < mid_end - mid_start) - return total; - - /* Unaligned tail [mid_end, end): through. */ - res = fuse_dlm_write_chunk(iocb, from, file, end - mid_end, true); - if (res > 0) + /* + * Every chunk reports a failure the same way: the error while + * nothing has landed, a short write once something has. Returning + * what has landed when that is nothing reports no error and no + * bytes, which fuse_cache_write_iter() turns into the full count. + */ + for (i = 0; i < ARRAY_SIZE(chunk); i++) { + res = fuse_dlm_write_chunk(iocb, from, file, chunk[i].len, + chunk[i].through); + if (res < 0) + return total ? total : res; total += res; + if (res < chunk[i].len) + break; + } return total; } @@ -1902,19 +2187,13 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) * fc->dlm: the server has no DLM, proceed as a plain cached write. Any * other failure means the cache would be dirtied without DLM coverage - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed, but flag it - * so the in-gate re-validation skips a check an invisible grant could - * never pass. + * lock (positive return) is covered cluster-wide; proceed. */ -static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, - bool *unrecorded) +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) { int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); - if (err < 0 && err != -ENOSYS) - return err; - *unrecorded = err > 0; - return 0; + return (err < 0 && err != -ENOSYS) ? err : 0; } static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -1927,13 +2206,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; bool writeback = false; - bool wb_guard = false; - bool exclusive = true; - bool dlm_unrecorded = false; - loff_t dlm_pos = 0; - size_t dlm_len = 0; + bool exclusive; if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -1977,98 +2251,64 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) writeback = true; } - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); - /* * Request the DLM write lock before taking i_rwsem: the request is * an unbounded cluster round trip, and holding the writer-priority * rwsem across it would park a truncate -- and behind it every - * later writer -- for the duration. The grant-to-use window this - * leaves open is closed by the in-gate re-validation below. Only - * the append case must wait for the lock: its range depends on - * i_size, which is stable only under the exclusive inode lock. + * later writer -- for the duration. Only the append case must wait + * for the lock: its range depends on i_size, which is settled by + * generic_write_checks() under the exclusive inode lock. + * + * The request may find that the server has no DLM at all and clear + * fc->dlm, so pick the lock mode after it rather than before. The + * relaxed shared lock is only sound under DLM: the shared path + * claims the i_size extension up front, which stops iomap from + * zeroing beyond EOF, and the zero-fill that replaces it in + * fuse_iomap_read_folio_range() is itself gated on fc->dlm. Chosen + * too early, an expanding write would fall through to a READ of a + * range that cannot hold data -- which fails outright on a handle + * the client opened write-only. */ if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { - dlm_pos = iocb->ki_pos; - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, + iov_iter_count(from)); if (err) return err; - - /* - * The request above may have found that the server has no DLM - * at all, in which case it cleared fc->dlm. The relaxed shared - * lock was chosen just before, while fc->dlm still read 1, and - * it is only sound under DLM: the shared path claims the i_size - * extension up front, which stops iomap from zeroing beyond - * EOF, and the zero-fill that replaces it in - * fuse_iomap_read_folio_range() is itself gated on fc->dlm. - * Left as chosen, an expanding write would fall through to a - * READ of a range that cannot hold data -- which fails outright - * on a handle the client opened write-only. Re-decide now, - * while no lock is held yet. - */ - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* note that this small code dup will save us a lot of headache later - * when appends are done concurrently without using parallel direct writes */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { - /* - * An append write lands at the current EOF no matter what - * ki_pos holds: generic_write_checks() rewrites ki_pos to - * i_size for IOCB_APPEND. Lock where the data will land. - */ - dlm_pos = i_size_read(inode); - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) - goto out; - } - err = count = generic_write_checks(iocb, from); if (err <= 0) goto out; /* - * The exclusive inode lock does not pin i_size for the append: - * attribute replies move it under fi->lock alone, so - * generic_write_checks() may have put ki_pos past the granted - * range. Re-lock where the write really lands; dlm_pos tracks it - * so the in-gate re-validation below guards the same range. + * An append lands at the EOF generic_write_checks() has just written + * into ki_pos, not where the caller pointed, and the exclusive inode + * lock does not pin i_size either: attribute replies move it under + * fi->lock alone. Take the grant here, where the range is settled. */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND) && - iocb->ki_pos != dlm_pos) { - dlm_pos = iocb->ki_pos; - dlm_len = count; - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; } /* - * Kill suid/sgid and stamp the timestamps here, before the gate, - * instead of leaving them next to the write itself. kiocb_modified() - * -> file_remove_privs() is the one that reaches the server: without - * handle_killpriv[_v2] fuse_setattr() kills the bits by asking it (a - * FUSE_GETATTR to refresh the mode, then a FUSE_SETATTR, which for a - * writeback inode first flushes and freezes writepages), and - * security_inode_killpriv() can drop the capability xattr with another - * round trip. A server may have to invalidate this inode from inside - * such a handler; its NOTIFY_INVAL_INODE then blocks in - * percpu_down_write() draining a gate reader that is itself waiting for - * the reply. Nothing held under the gate may wait for the server. + * Kill suid/sgid and stamp the timestamps here, ahead of the write + * itself. kiocb_modified() -> file_remove_privs() is the one that + * reaches the server: without handle_killpriv[_v2] fuse_setattr() + * kills the bits by asking it (a FUSE_GETATTR to refresh the mode, + * then a FUSE_SETATTR, which for a writeback inode first flushes and + * freezes writepages), and security_inode_killpriv() can drop the + * capability xattr with another round trip. A server may have to + * invalidate this inode from inside such a handler, and it must not + * find this write holding anything it needs. * * This also runs before the forced-DIO re-route below, so a re-routed * write repeats it; there is nothing left to do the second time. @@ -2077,31 +2317,32 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) goto out; - wb_guard = !!wb_sem; - if (wb_guard) { -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - fuse_cache_wr_unlock(inode, exclusive); - return fuse_direct_write_iter(iocb, from); - } - if (writeback && fc->dlm && !dlm_unrecorded && - !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, - FUSE_PAGE_LOCK_WRITE)) { - percpu_up_read(wb_sem); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) { - /* The gate is already dropped; funnel the - * failure through the one audited exit. */ - wb_guard = false; - goto out; - } - goto retry; - } + if (fuse_inode_force_dio(inode)) { + /* + * As on the read side, only worse: the direct write would + * land under whatever a write racing the latch left dirty, + * and the invalidate fuse_direct_write_iter() does after it + * launders rather than drops, putting that folio on the + * server on top. Send it first and the order is ordinary. + */ + if (count) + err = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + fuse_cache_wr_unlock(inode, exclusive); + if (err) + return err; + return fuse_direct_write_iter(iocb, from); } + /* + * A NOTIFY invalidate can revoke the grant requested above between + * here and the dirtying below, and nothing stops it: the bytes are + * caught on the way out instead. fuse_dlm_unlock_range() keeps a + * revoked range for as long as there is page cache under it, and + * writeback holds the range again before sending anything. So a + * write racing a revoke costs a round trip, not coverage. + */ + task_io_account_write(count); if (iocb->ki_flags & IOCB_DIRECT) { @@ -2141,12 +2382,39 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) spin_lock(&fi->lock); orig_size = i_size_read(inode); if (end > orig_size) { + /* + * Retire the attribute replies already on the + * wire. fuse_attr_cache_mask() decides whether + * the server's size wins from an i_size it read + * before this claim and before it slept in the + * lock tree query, so a GETATTR that left while + * i_size still matched the server's is applied + * afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() + * drop those replies, which is what + * fuse_write_update_attr() moves it for. + */ + fi->attr_version = + atomic64_inc_return(&fc->attr_version); + /* + * And count the claim, for a reply that leaves + * after it: until the bytes are dirtied, + * [orig_size, end) is covered by nothing else + * fuse_attr_cache_mask() can see. + */ + atomic_inc(&fi->size_extenders); i_size_write(inode, end); extended = true; } spin_unlock(&fi->lock); - /* Zero the tail of the folio straddling the old EOF. */ + /* + * Zero the tail of the folio straddling the old EOF. + * Inert while the fuse block size is PAGE_SIZE, which + * it always is, and nothing is recorded for it either + * way: claiming the whole gap as written would hand + * writeback bytes no one wrote. + */ if (extended && orig_size < pos) pagecache_isize_extended(inode, orig_size, pos); } @@ -2154,7 +2422,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * Under DLM the unaligned edges go through to the server * instead of being completed by a read-modify-write READ - * (see fuse_dlm_buffered_write()); only whole pages are + * (see fuse_dlm_buffered_write()); only whole blocks are * cached for writeback. */ if (fc->dlm) @@ -2179,6 +2447,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) i_size_write(inode, reached); spin_unlock(&fi->lock); } + atomic_dec(&fi->size_extenders); } if (written < 0) { @@ -2189,8 +2458,6 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from, false); } out: - if (wb_guard) - percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -2549,6 +2816,25 @@ static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, return iter_file_splice_write(pipe, out, ppos, len, flags); } +/* + * The folios and descs of a writeback request come from one allocation, + * which kfree(ap->folios) releases. + */ +static struct folio **fuse_wb_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **descs) +{ + struct folio **folios; + + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc)), flags); + if (!folios) + return NULL; + + *descs = (void *) (folios + nfolios); + + return folios; +} + static void fuse_writepage_free(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2562,6 +2848,25 @@ static void fuse_writepage_free(struct fuse_writepage_args *wpa) kfree(wpa); } +/* + * Ending a folio's writeback is iomap's own accounting, and fuse charges + * each run to the folio as it is queued and reports it back as its request + * completes. + * + * For a folio of more than one block, iomap_writeback_folio() holds a bias + * on ifs->write_bytes_pending across the whole walk and drops it once the + * last run has been offered, so the count cannot reach zero while a run is + * still to come. + * + * For a folio of a single block there is no iomap_folio_state and + * iomap_finish_folio_write() ends the writeback on every call, but such a + * folio is offered as exactly one run: iomap_find_dirty_range() has no + * per-block state to walk and returns the whole range at once, and + * ->writeback_range never returns short, so iomap_writeback_range() does + * not go round again. One run is one entry in one request, and one call + * ends it. fuse_iomap_writeback_range() asserts that. + */ + static void fuse_writepage_finish(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2575,7 +2880,8 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) * scope of the fi->lock alleviates xarray lock * contention and noticeably improves performance. */ - iomap_finish_folio_write(inode, ap->folios[i], 1); + iomap_finish_folio_write(inode, ap->folios[i], + ap->descs[i].length); wake_up(&fi->page_waitq); } @@ -2723,7 +3029,7 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) if (wpa) { ap = &wpa->ia.ap; ap->num_folios = 0; - ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + ap->folios = fuse_wb_folios_alloc(1, GFP_NOFS, &ap->descs); if (!ap->folios) { kfree(wpa); wpa = NULL; @@ -2748,7 +3054,7 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, } static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, - uint32_t folio_index, loff_t offset, unsigned len) + uint32_t folio_index, loff_t offset, unsigned int len) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2783,16 +3089,44 @@ static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio return wpa; } +/* + * How many times a data integrity writeback goes round for runs it had to + * skip. Each pass takes the grants the one before it deferred, so one more + * is normally enough; the cap is there because a revoke can take them again. + */ +#define FUSE_WB_DEFER_PASSES 4 + struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; unsigned int max_folios; + /* + * The folio currently being split into runs, and whether any run of + * it has already been queued. Only a folio iomap tracks per block + * can be offered as more than one run; see fuse_writepage_finish(). + */ + struct folio *wb_folio; + bool wb_queued; /* * nr_bytes won't overflow since fuse_writepage_need_send() caps * wb requests to never exceed fc->max_pages (which has an upper bound * of U16_MAX). */ unsigned int nr_bytes; + /* + * The runs this pass could not send because their grant had gone. + * Taken back in fuse_iomap_writeback_submit(), where no folio is + * held, for the pass that follows; see fuse_iomap_writeback_range(). + */ + u64 regrant_start; + u64 regrant_end; + /* fuse_iomap_writeback_submit() took that range back */ + bool regranted; + /* + * A folio whose run was skipped, held by a reference until it can be + * put back on the dirty list; see fuse_writeback_redirty(). + */ + struct folio *redirty; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, @@ -2807,7 +3141,7 @@ static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, max_pages); WARN_ON(nfolios <= data->max_folios); - folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + folios = fuse_wb_folios_alloc(nfolios, GFP_NOFS, &descs); if (!folios) return false; @@ -2872,7 +3206,7 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, unsigned int total_pages = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; pgoff_t page_index = pos >> PAGE_SHIFT; - if (!(page_index % fc->alignment_pages)) { + if (wbc && !(page_index % fc->alignment_pages)) { pgoff_t end_page_index = (wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT; /* we are at a point where we would write aligned @@ -2887,6 +3221,62 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, return false; } +/* + * Put a folio writeback could not send back on the dirty list. + * + * iomap takes the dirty flag off a folio before it offers it to + * ->writeback_range, and a run reporting an error, or reporting a hole + * because the grant has gone, has thrown its bytes away unless they are put + * back. + * + * Not from inside the callback, though. iomap_writeback_folio() runs + * iomap_clear_range_dirty() over the whole folio once that has returned, so + * a range put back there is wiped again. For a folio one block wide that + * call does nothing and it would not matter, but a large folio carries an + * iomap_folio_state, and the folio would then be left with the dirty flag + * and no dirty block under it: the next pass finds nothing to write and the + * folio goes clean with its bytes never sent. + * + * So hold the folio and dirty it once iomap has let go of it, which on the + * ->writepages path is the next call or the submit, and in + * fuse_launder_folio() is the submit it makes itself. + * + * Only while there is a connection left to take the bytes. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty_done(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc) +{ + struct folio *folio = data->redirty; + + if (!folio) + return; + data->redirty = NULL; + + if (READ_ONCE(fc->connected)) { + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); + } + folio_put(folio); +} + +/* Remember @folio for fuse_writeback_redirty_done() */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc, + struct folio *folio) +{ + if (data->redirty == folio) + return; + + fuse_writeback_redirty_done(fc, data, wbc); + folio_get(folio); + data->redirty = folio; +} + static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct folio *folio, u64 pos, unsigned len, u64 end_pos) @@ -2897,16 +3287,154 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct inode *inode = wpc->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - loff_t offset = offset_in_folio(folio, pos); + struct fuse_dlm_span pin; + bool pinned = false; + loff_t offset; WARN_ON_ONCE(!data); if (!data->ff) { data->ff = fuse_write_file_get(fi); - if (!data->ff) + if (!data->ff) { + /* + * No file left open for writing, which + * fuse_open()'s invalidate reaches through + * fuse_launder_folio() once the last writer has + * closed. The bytes are still the newest there + * are, so keep them: dropping them here loses a + * write that fsync and close both reported done. + */ + fuse_writeback_redirty(fc, data, wpc->wbc, folio); return -EIO; + } + } + + /* + * A folio iomap has not asked about before: the one before it has all + * of its runs queued. + */ + if (data->wb_folio != folio) { + data->wb_folio = folio; + data->wb_queued = false; + /* + * iomap has unlocked whatever it offered before this, so a + * folio held from then can go back on the dirty list now. + */ + fuse_writeback_redirty_done(fc, data, wpc->wbc); } + /* + * Every run iomap reports dirty was written whole by this client: + * the unaligned edges of a cached write go to the server directly + * and the interior covers whole blocks, so nothing partly written + * is ever dirtied. There is nothing to classify, only the grant to + * make sure of: a revoke may have arrived since the write, and + * these bytes must not go out from under one. + * + * fuse_dlm_regrant_range() takes the range back when it has gone, + * and walks the record once under the lock held for read when it + * has not. A failure redirties the folio, so the next writeback + * tries again. A revoke handler driving this skips the run + * entirely rather than ask for the grant it is taking away. + * + * The grant is pinned over this run from the moment it is found + * until the folio is under writeback, where the revoke waits for it + * again. Without that these bytes could still go out under a + * revoke that has already been answered. + */ + if (fc->dlm && fc->writeback_cache) { + int err; + + /* + * wpc->iomap.type carries over from the previous run and from + * the previous folio, so anything that is queued has to say + * so. Left at a stale IOMAP_HOLE, iomap takes the folio for + * one it never queued and ends its writeback while the write + * is still in flight. + */ + wpc->iomap.type = IOMAP_MAPPED; + + /* + * The revoke handler flushing the range it is taking + * away. That lock is still this client's until the + * handler returns, so send without asking: the record has + * gone already and asking would be a round trip for the + * very range being revoked. Only for that range, since + * the latched path launders the whole mapping and the + * rest of it may be covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + + /* + * The folio is locked and under writeback here, so a grant + * this run does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. read_folio() has AOP_TRUNCATED_PAGE to unlock and + * retry with; ->writeback_range has nothing of the sort. + * + * Report the run as a hole instead. The folio goes back on + * the dirty list, the range is remembered for + * fuse_iomap_writeback_submit() to take back with no folio + * held, and the pass that follows sends it. Nothing is lost + * and no error is recorded for a later fsync to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the run in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. + */ + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + u64 s = min(data->regrant_start, pos); + u64 e = max(data->regrant_end, pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips runs a long way apart, and the + * span between them says nothing about what + * is wanted; the runs left out stay dirty and + * a later pass asks for them. + */ + if (e - s <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = s; + data->regrant_end = e; + } + } + wpc->iomap.type = IOMAP_HOLE; + return len; + } + + /* + * Held, and pinned so it stays held: this walks the record + * and sends nothing. It stays a call rather than the check + * above so a grant that arrives between them is still used. + */ + err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); + if (err < 0 && err != -ENOSYS) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + fuse_dlm_unpin(fi); + return err; + } + } +queue: + + offset = offset_in_folio(folio, pos); + if (wpa && fuse_writepage_need_send(fc, pos, len, ap, data, wpc->wbc)) { fuse_writepages_send(inode, data); data->wpa = NULL; @@ -2915,16 +3443,37 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (data->wpa == NULL) { wpa = fuse_writepage_args_setup(folio, offset, data->ff); - if (!wpa) + if (!wpa) { + fuse_writeback_redirty(fc, data, wpc->wbc, folio); + if (pinned) + fuse_dlm_unpin(fi); return -ENOMEM; + } fuse_file_get(wpa->ia.ff); data->max_folios = 1; ap = &wpa->ia.ap; } - iomap_start_folio_write(inode, folio, 1); - fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, - offset, len); + /* + * fuse_writepage_finish() reports this run's bytes when the request + * completes. A folio of a single block carries no iomap_folio_state + * and that report ends its writeback outright, which is only correct + * because iomap offers such a folio as exactly one run. Nothing + * above can make it offer a second, but the accounting depends on it. + */ + WARN_ON_ONCE(data->wb_queued && i_blocks_per_folio(inode, folio) == 1); + data->wb_queued = true; + iomap_start_folio_write(inode, folio, len); + + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, offset, len); data->nr_bytes += len; ap->num_folios++; @@ -2941,11 +3490,34 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, WARN_ON_ONCE(!data); + /* No more runs are coming for the folio last seen */ + data->wb_folio = NULL; + data->wb_queued = false; + fuse_writeback_redirty_done(get_fuse_conn(wpc->inode), data, wpc->wbc); + if (data->wpa) { WARN_ON(!data->wpa->ia.ap.num_folios); fuse_writepages_send(wpc->inode, data); } + /* + * Take back what the runs above had to skip, so the pass that + * follows finds the grant and sends the folios they left dirty. + * + * Only on the ->writepages path, which is what @wbc marks. + * fuse_launder_folio() reaches here with the folio still locked by + * folio_unmap_invalidate(), which is the ordering this is avoiding, + * and a revoke handler would ask for the very range it is revoking. + * In both the skip simply stands and writeback picks it up. + */ + if (wpc->wbc && data->ff && data->regrant_end > data->regrant_start && + !fuse_in_notify_ctx()) { + fuse_dlm_regrant_range(data->ff, wpc->inode, + data->regrant_start, + data->regrant_end - 1); + data->regranted = true; + } + if (data->ff) fuse_file_put(data->ff, false); @@ -2962,14 +3534,8 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data = {}; - struct iomap_writepage_ctx wpc = { - .inode = inode, - .iomap.type = IOMAP_MAPPED, - .wbc = wbc, - .ops = &fuse_writeback_ops, - .wb_ctx = &data, - }; + unsigned int tries = FUSE_WB_DEFER_PASSES; + int err; if (fuse_is_bad(inode)) return -EIO; @@ -2978,7 +3544,35 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - return iomap_writepages(&wpc); + /* + * A run whose grant had gone is skipped and its range taken back in + * fuse_iomap_writeback_submit(), which leaves the folio dirty for a + * later pass. For a data integrity writeback there is no later + * pass: fsync() and close() would report the bytes written while + * they are still only in the page cache. Go round again, now that + * the grant is held, until nothing is left deferred. + */ + do { + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = inode, + .iomap.type = IOMAP_MAPPED, + .wbc = wbc, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + err = iomap_writepages(&wpc); + /* + * Only where the submit took the grant back. A revoke + * handler driving this is not allowed to, so the runs it + * skipped would be skipped again by every pass. + */ + if (err || wbc->sync_mode != WB_SYNC_ALL || !data.regranted) + break; + } while (--tries); + + return err; } static int fuse_launder_folio(struct folio *folio) @@ -3065,6 +3659,7 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l fuse_abort_conn(fc); err = -EINVAL; } + return err; } /* @@ -3090,9 +3685,13 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct fuse_mount *fm = get_fuse_mount(inode); if (fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; - size_t length = PAGE_SIZE; - int err = fuse_get_page_mkwrite_lock(file, pos, length); + /* + * The whole folio is dirtied on the way out of this fault + * (fault_dirty_shared_page()), so the lock has to cover the + * folio, not the page that faulted. + */ + int err = fuse_get_page_mkwrite_lock(file, folio_pos(folio), + folio_size(folio)); if (err < 0) { return vmf_error(err); } @@ -3140,7 +3739,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) /* * If the inode was latched into forced direct IO after a remote-modify * notification, a mapping needs the page cache, so revert to caching - * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * mode. Revert without the inode lock: ->mmap runs * under mmap_lock and the buffered write path holds both across a fault * on the user buffer (which takes mmap_lock), so taking either here * would invert lock order (ABBA). Clearing the latch and dropping the @@ -3984,25 +4583,9 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - /* - * Coherency gate for the forced-direct-IO feature; only writeback+dlm - * regular files need it. A percpu_rw_semaphore embeds per-CPU state, - * so allocate it out of line and only when the mount can use it rather - * than paying it on every inode. On failure leave it NULL: the gate - * stays inactive (best-effort invalidate) and the inode is still usable. - */ - fi->wb_inval_rwsem = NULL; - if (fc->writeback_cache && fc->dlm) { - struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); - - if (sem && percpu_init_rwsem(sem)) { - kfree(sem); - sem = NULL; - } - fi->wb_inval_rwsem = sem; - } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; + atomic_set(&fi->size_extenders, 0); if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index bc6dbae2d5aeb0..d3ef8548d3b380 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -1,596 +1,780 @@ // SPDX-License-Identifier: GPL-2.0-only /* * FUSE page lock cache implementation + * + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is + * cached there, and whether the server has seen it, is what the page + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include +#include #include #include -#include +#include +#include -/* A range of pages with a lock */ +/* + * How often to ask again for a grant a revoke killed while it was in + * flight, or the server refused as contended, before giving up on the + * range. Each pass is a round trip. + */ +#define FUSE_DLM_GRANT_RETRIES 16 + +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) + +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { - /* Interval tree node */ - struct rb_node rb; - /* Start page offset (inclusive) */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; - /* End page offset (inclusive) */ uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lock mode */ - enum fuse_page_lock_mode mode; - /* Temporary list entry for operations */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ + bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; + /* The cache->pending link */ struct list_head list; }; -/* Lock modes for FUSE page cache */ -#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ -#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ - -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) -{ - return range->start; -} - -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return range->end; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); - /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at * - * Initialize a page cache lock manager for a FUSE inode. + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. * - * Return: 0 on success, negative error code on failure + * Return: the shard, or NULL if it could not be allocated. */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; - if (!cache) - return -EINVAL; + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; - init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; - cache->revoke_gen = 0; + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; - return 0; + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } + + return shard; } /** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up + * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] + * @cache: The page cache + * @start: Start byte offset of the revoked region + * @end: End byte offset of the revoked region * - * Release all locks and free all resources associated with the cache. + * A revoke overlapping a request still on the wire has nothing to remove + * from the tree, since that grant is not recorded yet. Marking it makes + * fuse_dlm_request_commit() drop the grant instead of recording it. + * + * The nodes are owned by the threads waiting on their replies: mark + * only, never remove or free. + * + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ -void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, + uint64_t start, uint64_t end) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; - - if (!cache) - return; - - /* Release all locks */ - down_write(&cache->lock); - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); + struct fuse_dlm_range *req; + + spin_lock(&cache->pending_lock); + list_for_each_entry(req, &cache->pending, list) { + if (req->start <= end && start <= req->end) + req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; } - up_write(&cache->lock); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * - * Return: Pointer to the first overlapping range, or NULL if none found + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_cache_init(struct fuse_inode *inode) { - return fuse_page_it_iter_first(&cache->ranges, start, end); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + init_rwsem(&cache->lock); + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); + INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); } -/** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = current; +} - if (!cache) - return; +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. + */ +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; - /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. - */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; - /* Try to merge with next range if adjacent and same mode */ - if (next && range->mode == next->mode && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } + return false; +} - /* Move to next range */ - range = next; - } +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) +{ + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; } /** - * __fuse_dlm_lock_range - Lock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode (read or write) - * @genp: If non-NULL, the revocation generation sampled before the grant - * was requested; recording fails with -EAGAIN if it has moved - * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * fuse_dlm_pin - hold the grants over a range until fuse_dlm_unpin() + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes * - * Return: 0 on success, negative error code on failure + * Waits out a revoke overlapping that range, so it must not be called + * with a folio held: the revoke drops that same page cache once it has + * drained. A revoke elsewhere in the file is not waited for. The caller + * confirms its grant after this returns, never before; a confirmation + * from before the pin says nothing. */ -static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - const uint64_t *genp) +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - int lock_mode; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; - - if (!cache || start > end) - return -EINVAL; - /* Convert to lock mode */ - lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; + /* + * A revoke handler driving this inode's page cache: the lock over + * the range is still this client's until the handler returns, and + * the fence this would wait on is the handler's own. Both ends + * test the same task, so nothing is left on the list. + */ + if (fuse_in_notify_ctx()) + return; - down_write(&cache->lock); + fuse_dlm_span_set(pin, offset, length); + spin_lock(&cache->pin_lock); + while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { + spin_unlock(&cache->pin_lock); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); + spin_lock(&cache->pin_lock); + } /* - * A revoke was processed after @genp was sampled; the grant this - * record carries may be the very one it targeted (a revoke of a - * not-yet-recorded grant removes nothing and would never be - * retried). Refuse, the caller re-requests. + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. */ - if (genp && cache->revoke_gen != *genp) { - up_write(&cache->lock); - return -EAGAIN; - } + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); +} - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check lock compatibility */ - if (lock_mode == FUSE_PCACHE_LK_WRITE && - lock_mode != range->mode) { - /* we own the lock but have to update it. */ - list_add_tail(&range->list, &to_upgrade); - } - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } +/** + * fuse_dlm_trypin - fuse_dlm_pin() for a caller that cannot sleep + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * For the writeback path, which holds a folio locked and under writeback + * and has nothing to wait with. A refusal means a revoke of this range + * is draining; the caller redirties and the pass that follows sends the + * folio. + * + * Return: true if the range is pinned, false if it is not. + */ +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return true; - /* Move to next range */ - range = next; - } + fuse_dlm_span_set(pin, offset, length); - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); - new_range->start = current_start; - new_range->end = end; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); + return !fenced; +} - list_add_tail(&new_range->list, &to_lock); - } +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; - /* update locks, if any lock is in this list it has the wrong mode */ - list_for_each_entry(range, &to_upgrade, list) { - /* Update the lock mode */ - range->mode = lock_mode; - } + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return; - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; + } } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); + if (waiters) + wake_up_all(&cache->pin_wq); +} - up_write(&cache->lock); - return 0; +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; - /* Restore original lock modes for any partially upgraded locks */ - list_for_each_entry(range, &to_upgrade, list) { - if (lock_mode == FUSE_PCACHE_LK_WRITE) { - /* We upgraded this lock but failed later, downgrade it back */ - range->mode = FUSE_PCACHE_LK_READ; - } - } + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); - up_write(&cache->lock); - return ret; + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); } -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) { - return __fuse_dlm_lock_range(inode, start, end, mode, NULL); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); + + wake_up_all(&cache->pin_wq); } -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen) +/** + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of + * + * Release all locks and free all resources associated with the cache. + */ +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { - return __fuse_dlm_lock_range(inode, start, end, mode, &gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_shard *shard; + unsigned long idx; + + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); + + xa_for_each(&cache->shards, idx, shard) + kfree(shard); + /* + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. + */ + xa_destroy(&cache->shards); + up_write(&cache->lock); } /** - * fuse_dlm_revoke_gen - sample the revocation generation - * @inode: the fuse inode + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in * - * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The - * reply and a NOTIFY revoke can be serviced on different threads, so a - * revoke may be processed between the reply arriving and its grant - * being recorded. fuse_dlm_lock_range_gen() re-checks the generation - * under the cache lock and refuses to record a grant such a revoke may - * have already killed. + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. + * + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. + * + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - return READ_ONCE(inode->dlm_locked_areas.revoke_gen); + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); + + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); + } } /** - * fuse_dlm_punch_hole - Punch a hole in a locked range - * @cache: The page cache - * @start: Start page offset of the hole - * @end: End page offset of the hole + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in * - * Create a hole in a locked range by splitting it into two ranges. + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. * - * Return: 0 on success, negative error code on failure + * Caller holds @cache->lock for read. + * + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_range *range, *new_range; - int ret = 0; + unsigned long idx, last_idx; - if (!cache || start > end) + if (start > end) return -EINVAL; - /* Find a range that contains [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - if (!range) { - ret = -EINVAL; - goto out; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* If the hole is at the beginning of the range */ - if (start == range->start) { - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - /* If the hole is at the end of the range */ - if (end == range->end) { - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; - /* The hole is in the middle, need to split */ - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out; + fuse_dlm_shard_record(shard, lo, hi, mode); } - /* Copy properties from original range */ - *new_range = *range; - INIT_LIST_HEAD(&new_range->list); - - /* Adjust ranges */ - new_range->start = end + 1; - range->end = start - 1; + return 0; +} - /* Update interval tree */ - fuse_page_it_remove(range, &cache->ranges); - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(new_range, &cache->ranges); +/** + * fuse_dlm_request_begin - publish a lock request before it is sent + * @inode: the fuse inode + * @req: caller-owned storage for the request, live until commit or abort + * @start: start byte offset being requested (inclusive) + * @end: end byte offset being requested (inclusive) + * + * The mode is not recorded here: until the server answers the range is + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). + * + * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different + * threads, so a revoke can be processed before the grant the reply + * carries is recorded. Publishing the request before it leaves gives + * that revoke a node to mark; without one it removes nothing, and the + * grant recorded afterwards is never taken back by any later NOTIFY. + * + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. + * + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; -out: - return ret; + req->start = start; + req->end = end; + /* + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. + */ + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; + req->killed = false; + req->clamp = false; + + spin_lock(&cache->pending_lock); + list_add_tail(&req->list, &cache->pending); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_unlock_range - Unlock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_request_commit - retire a request and record its grant + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode that was requested * - * Release locks on the specified range of pages. An inverted range is - * rejected rather than silently removing nothing: the callers revoke - * coverage, and a revoke that quietly keeps the grant alive would let - * the re-validating IO paths trust a lock the server has taken away. - * To drop every grant use fuse_dlm_cache_release_locks() (there is no - * in-band sentinel range for it). + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. * - * Return: 0 on success, negative error code on failure + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. + * + * @req is retired in every case and may be reused. + * + * Return: -EAGAIN if a revoke overlapped the range @req asked for, + * nothing recorded; otherwise the result of recording the grant. */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; + bool revoked, clamp; int ret = 0; - if (!cache || start > end) - return -EINVAL; - - down_write(&cache->lock); - /* - * Unconditional, even when nothing overlaps: the revoke racing - * with an in-flight grant finds an empty tree precisely because - * the grant is not recorded yet, and the bump is what makes the - * recording side notice (see fuse_dlm_lock_range_gen()). + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. */ - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check if we need to punch a hole */ - if (start > range->start && end < range->end) { - /* Punch a hole in the middle */ - ret = fuse_dlm_punch_hole(cache, start, end); - if (ret) - goto out; - /* After punching a hole, we're done */ - break; - } else if (start > range->start) { - /* Adjust the end of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - } else if (end < range->end) { - /* Adjust the start of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - } else { - /* Complete overlap, remove the range */ - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } + down_read(&cache->lock); - range = next; + spin_lock(&cache->pending_lock); + list_del(&req->list); + revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; } + spin_unlock(&cache->pending_lock); -out: - up_write(&cache->lock); - return ret; + if (!revoked) + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); + + return revoked ? -EAGAIN : ret; } /** - * fuse_dlm_range_is_locked - Check if a page range is already locked - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode to check for (or NULL to check for any lock) + * fuse_dlm_request_abort - retire a request that got no usable reply + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * + * Nothing is recorded, so a mark left by a revoke does not matter. + */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pending_lock); + list_del(&req->list); + spin_unlock(&cache->pending_lock); +} + +/** + * fuse_dlm_unlock_range - Revoke the grants over a range of pages + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset * - * Check if the specified range of pages is already locked. - * The entire range must be locked for this to return true. + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. * - * Return: true if the entire range is locked, false otherwise + * An inverted range is rejected rather than silently revoking nothing: + * the callers revoke coverage, and a revoke that quietly keeps the grant + * alive would let the re-validating IO paths trust a lock the server has + * taken away. To drop every grant use fuse_dlm_cache_release_locks() + * (there is no in-band sentinel range for it). + * + * Return: 0 on success, negative error code on failure */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - int lock_mode = 0; - uint64_t current_start = start; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) - return false; + if (start > end) + return -EINVAL; - /* Convert to lock mode if specified */ - if (mode == FUSE_PAGE_LOCK_READ) - lock_mode = FUSE_PCACHE_LK_READ; - else if (mode == FUSE_PAGE_LOCK_WRITE) - lock_mode = FUSE_PCACHE_LK_WRITE; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); - down_read(&cache->lock); + /* + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. + */ + fuse_dlm_kill_pending(cache, start, end); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; - /* Check if the entire range is covered */ - while (range && current_start <= end) { /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. */ - if (lock_mode && range->mode < lock_mode) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } - - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; + /* + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. + */ + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); } - - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); } - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } + up_write(&cache->lock); + return 0; +} - up_read(&cache->lock); - return true; +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. + * + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; } /** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode + * fuse_dlm_range_is_locked - Check if a byte range is already locked + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * @mode: Lock mode to check for * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. - * - * Return: true if at least one recorded range is held for write + * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx, last_idx; + bool covered = true; + if (start > end) + return false; + + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->mode == FUSE_PCACHE_LK_WRITE) { - held = true; + + last_idx = end >> FUSE_DLM_SHARD_SHIFT; + + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -634,11 +818,10 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, * re-validating the grant must not re-request on a nonzero return or * they would spin. */ -int fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) { - struct fuse_file *ff = file->private_data; - struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; @@ -646,13 +829,24 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; - uint64_t gen; + struct fuse_dlm_range req; + uint64_t pg_start, pg_end; + uint64_t grant_start, grant_end; + int tries = FUSE_DLM_GRANT_RETRIES; int err; /* An empty range needs no lock. */ if (!length) return 0; + /* + * note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests + */ + pg_start = (uint64_t)offset & PAGE_MASK; + pg_end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected @@ -661,27 +855,20 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * The early exit uses the same helper the callers re-validate * with, so this check and a later fuse_dlm_lock_is_held() can * never disagree about what counts as covered. */ - if (fuse_dlm_lock_is_held(fi, offset, length, mode)) - return 0; /* we already have this area locked */ - - /* - * Sample the revocation generation before the request leaves. - * The reply and a NOTIFY revoke are serviced on different - * threads, so a revoke aimed at the grant this request returns - * can be processed before the grant is recorded below -- - * recording it anyway would resurrect a dead grant that no later - * NOTIFY will ever remove. - */ - gen = fuse_dlm_revoke_gen(fi); + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) { + /* + * Already covered, and the record says nothing beyond that, + * so this is one shared acquisition end to end. + */ + return 0; + } memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned - * here but since we only get here on writeback caching we will - * send out page aligned requests */ - inarg.start = offset & PAGE_MASK; - inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.start = pg_start; + inarg.end = pg_end; inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -693,18 +880,33 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; + + /* Publish before sending; see fuse_dlm_request_begin() */ + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end); + err = fuse_simple_request(fm, &args); - if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; + if (err) { + fuse_dlm_request_abort(fi, &req); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + return err; + } + /* + * The range is contended, the same answer a READ gets and + * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * caller to retry. There is no such convention here, and + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. + */ + if (err == -EDEADLK || err == -EAGAIN) + goto retry; return err; } - if (err) - return err; - if (inarg.start < outarg.start || inarg.end > outarg.end) { /* fuse server is seriously broken */ + fuse_dlm_request_abort(fi, &req); pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); @@ -712,22 +914,45 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, } /* - * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. + */ + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. + */ + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. */ - err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* Retire the request and record the grant */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); if (err == -EAGAIN) { /* - * A revoke was processed while the request was in flight; - * the grant may already be dead, so re-request instead of - * recording it. Retry until a grant survives long enough to - * be recorded: giving up here would hand the caller an error - * for a range no one else holds, and the write path turns - * that into a failed write. Each pass makes a fresh server - * round trip, so a revoke storm throttles this loop rather - * than spinning it. + * A revoke overlapping this range was processed while the + * request was in flight, so the grant is dead. Retry + * rather than fail: no one else holds the range, and the + * write path turns an error into a failed write. */ - goto restart; + goto retry; } /* @@ -742,4 +967,47 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, return FUSE_DLM_GRANT_UNRECORDED; return 0; + +retry: + /* + * Ask again, but not forever. Every pass is a whole round trip, + * which throttles the loop but does not end it, and writeback asks + * for a grant with a folio locked, so a node taking the range as + * fast as this asks for it would hold that folio and this task for + * as long as it kept going. + */ + if (fatal_signal_pending(current)) + return -EINTR; + if (!tries--) + return -EIO; + goto restart; +} + +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return __fuse_get_dlm_lock(file->private_data, file_inode(file), + offset, length, mode); +} + +/** + * fuse_dlm_regrant_range - hold [start, end] again for writeback + * @ff: a fuse file open for writing on @inode + * @inode: the inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * Writeback holds the range again before sending a folio, since a revoke + * may have arrived between the write and the send. Whatever the other + * holder wrote in between is overwritten, which for two writers that + * never synchronised is a legitimate order. + * + * A range still held is the ordinary case: the grant is found recorded + * and nothing is sent to the server. + */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end) +{ + return __fuse_get_dlm_lock(ff, inode, start, end - start + 1, + FUSE_PAGE_LOCK_WRITE); } diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 30fdbb26bd3daf..ec2e82c406a1cb 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,16 +7,37 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include +#include struct fuse_inode; +struct fuse_dlm_range; +struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: a + * writer between confirming a grant and dirtying under it (@owner set), + * or a revoke taking grants away (@owner NULL). Caller-owned storage, + * live until the matching unpin or revoke end. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -26,53 +47,189 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 -/* Page cache lock manager */ +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + +/* + * Page cache lock manager. + * + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. + */ struct fuse_dlm_cache { - /* Lock protecting the tree */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of locked ranges */ - struct rb_root_cached ranges; /* - * Bumped under @lock by every revocation - * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); - * lets fuse_get_dlm_lock() order recording a reply's grant - * against revokes processed while the reply was in flight. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. */ - uint64_t revoke_gen; + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. + */ + struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * Writers between confirming a grant and dirtying under it, each + * over the range it is about to write. See the pin comment above. + */ + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a + * revoke processed while the reply is on the wire can mark it. @req is + * caller-owned storage, live until the matching commit or abort. The + * mode is not recorded until the grant is, so only the commit takes it. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end); -/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen); +/* + * Retire @req and record the grant [start, end] as one step under the + * cache lock. -EAGAIN means a revoke overlapped @req in flight and + * nothing was recorded; the caller must request again. @req is retired + * either way. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); -/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); +/* Retire @req without recording anything */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req); /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); + +/* + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* Hold [start, end] again so writeback can send what it found revoked */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 31580b7834e296..0fae65e89fbe7d 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,8 +32,71 @@ #include #include #include +#include #include "fuse_dlm_cache.h" +/* + * Page cache work driven by a NOTIFY invalidate is marked on the task. + * + * Writeback reached from there must not ask the server for a grant: the + * range is the one the server is revoking, and the request would go out + * from inside the handler the server is waiting on, with the folio locked + * and under writeback. See fuse_reverse_inval_inode() and + * fuse_iomap_writeback_range(). + */ +extern const char fuse_notify_ctx_key[]; + +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) +{ + void *old = current->journal_info; + + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; + return old; +} + +static inline void fuse_notify_ctx_leave(void *old) +{ + current->journal_info = old; +} + +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + +static inline bool fuse_in_notify_ctx(void) +{ + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; +} + /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -202,21 +265,6 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; - /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. - */ - struct percpu_rw_semaphore *wb_inval_rwsem; - /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations * for this whole file: notify_stamp is the jiffies of @@ -229,6 +277,20 @@ struct fuse_inode { */ unsigned long notify_stamp; unsigned int notify_interval_ewma; + + /* + * Buffered writes that have claimed an i_size + * extension and not yet dirtied it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; }; /* readdir cache (directory only) */ @@ -701,17 +763,6 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; -/** - * DLM retry tracking for iomap write deadlock workaround. - * - * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. Tracks tasks that need to retry write operations due - * to DLM lock contention (-EAGAIN from FUSE server). - */ -struct fuse_dlm_retry { - bool retry_needed; -}; - /** * A Fuse connection. * @@ -1100,13 +1151,6 @@ struct fuse_conn { /* The foffset alignment in PAGE */ unsigned int alignment_pages; - - /** - * XArray tracking tasks that need DLM retry. - * Maps task pointer -> struct fuse_dlm_retry. - * Temporary workaround for iomap write deadlock. - */ - struct xarray dlm_retry_tasks; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d54676b73abf9e..a58c1524617899 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,7 +37,14 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); -bool __read_mostly enable_large_folios = true; +/* + * A folio wider than a block carries per-block dirty state, which + * iomap_writeback_folio() clears whole after ->writeback_range. A run + * fuse defers and puts back with folio_mark_dirty() comes back dirty over + * the whole folio, so the next pass sends blocks this client never wrote. + * Off until there is a way to restore only the blocks that were dirty. + */ +bool __read_mostly enable_large_folios; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support"); @@ -220,23 +227,6 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } - - /* - * Free the coherency gate here rather than in ->free_inode: that runs - * from an RCU callback, where percpu_free_rwsem() may sleep in - * rcu_sync_dtor() if the write side has not fully quiesced. No user - * can remain by eviction time: gate readers hold a file reference and - * a concurrent notify holds an inode reference. wb_inval_rwsem lives - * in the regular-file union arm and is only ever allocated for regular - * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked - * regular files still own a gate); a directory's overlapping - * readdir-cache fields must not be misread. - */ - if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { - percpu_free_rwsem(fi->wb_inval_rwsem); - kfree(fi->wb_inval_rwsem); - fi->wb_inval_rwsem = NULL; - } } static int fuse_reconfigure(struct fs_context *fsc) @@ -371,8 +361,17 @@ static void fuse_change_attributes_common_sx(struct inode *inode, } } - /* Common fields for both statx and getattr */ - if (attr->blksize != 0) + /* + * Common fields for both statx and getattr. + * + * A writeback connection was refused at FUSE_INIT unless its block + * is a page, for the reasons given there, and a server naming a + * different one per inode does not get to take that back. What it + * named is still reported as st_blksize out of + * fi->cached_i_blkbits; this is only what the page cache is + * tracked in. + */ + if (attr->blksize != 0 && !fc->writeback_cache) inode->i_blkbits = ilog2(attr->blksize); else inode->i_blkbits = inode->i_sb->s_blocksize_bits; @@ -527,14 +526,14 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is entirely under a write grant. + * Taking the server's answer would shrink i_size and have + * truncate_pagecache() throw the unwritten tail away. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, @@ -558,16 +557,38 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, !S_ISREG(inode->i_mode)) return cache_mask; - if (!fuse_dlm_write_grant_exists(fi)) - return cache_mask; - + /* + * A dirty mapping keeps the local attributes authoritative even + * when no grant is recorded: a fault dirties pages under a + * page-mkwrite lock that is never recorded, and a truncate revokes + * the tail grants itself while cached writes above the new size + * are still waiting for writeback. + */ if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; + /* + * The local size stays authoritative while the extension is + * covered by a write grant, and also while anything in + * [attr->size, size) is dirty or under writeback: those bytes + * exist only here, and taking the server's smaller size would + * truncate them away before they are ever sent. The grant check + * alone misses them, because a page-mkwrite grant is never + * recorded and a local truncate revokes its own tail grants. + * + * Both miss a write that has claimed its extension and not yet + * dirtied it: nothing is dirty there, and a NOTIFY can revoke the + * grant in between. With i_rwsem held shared several writers sit + * in that window at once, which is why they are counted rather + * than flagged. + */ if (have_size && size > (loff_t) attr->size && - fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE)) + (atomic_read(&fi->size_extenders) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE) || + filemap_range_needs_writeback(inode->i_mapping, attr->size, + size - 1))) cache_mask |= STATX_SIZE; return cache_mask; @@ -878,6 +899,9 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* Address only; see fuse_notify_ctx_enter() */ +const char fuse_notify_ctx_key[1]; + /* * Revoke the DLM grants backing an invalidated byte range. Grants are * recorded page-aligned, so widen the revoke to page boundaries: dropping @@ -901,44 +925,63 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * Drop a page-cache range on behalf of a NOTIFY invalidate. * * invalidate_inode_pages2_range() waits out folios under writeback and - * launders dirty ones, both of which need a FUSE_WRITE reply. While - * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, - * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() - * parks the request on fi->queued_writes until fuse_release_nowrite(). A - * server that revokes from inside the handler it is revoking for then - * deadlocks against its own reply. fuse_do_setattr() states the same rule - * for its own invalidate. + * launders dirty ones, both of which need a FUSE_WRITE reply. It is only + * needed when the range can hold data the server has not seen. + * + * @may_be_dirty false says it cannot, on the strength of the DLM range + * record: every way a folio gets dirtied under a grant raises that record + * before the data lands, so a range it reports clean has no dirty folio to + * launder. invalidate_mapping_pages() then drops the same folios without + * ever waiting for the server. * - * So while frozen use invalidate_mapping_pages(), which skips dirty and - * under-writeback folios and never blocks. The stale clean folios still - * go, and the freezes that span a request drop the cache themselves once - * they complete: fuse_do_setattr() invalidates the mapping after releasing - * the freeze, the O_TRUNC open path calls truncate_pagecache(). + * The same substitution is forced while writepages are frozen + * (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, pre-SETATTR flush), + * where no reply can arrive because fuse_flush_writepages() parks the + * request on fi->queued_writes until fuse_release_nowrite(). A server that + * revokes from inside the handler it is revoking for would otherwise + * deadlock against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. There the dirty folios are left behind, and the + * freezes that span a request drop the cache themselves once they complete: + * fuse_do_setattr() invalidates the mapping after releasing the freeze, the + * O_TRUNC open path calls truncate_pagecache(). + * + * A drop that can launder writes the range back itself first. + * fuse_launder_folio() is handed one folio at a time and sends a FUSE_WRITE + * for each, where a writeback pass batches the same bytes up to + * fc->max_write; what it sends the drop then only waits for. */ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); + loff_t last; bool frozen; spin_lock(&fi->lock); frozen = fi->writectr < 0; spin_unlock(&fi->lock); - if (frozen) + if (frozen || !may_be_dirty) { invalidate_mapping_pages(inode->i_mapping, start, end); - else - invalidate_inode_pages2_range(inode->i_mapping, start, end); + return; + } + + last = end == (pgoff_t)-1 ? LLONG_MAX : + (((loff_t)end + 1) << PAGE_SHIFT) - 1; + filemap_write_and_wait_range(inode->i_mapping, + (loff_t)start << PAGE_SHIFT, last); + invalidate_inode_pages2_range(inode->i_mapping, start, end); } int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; + bool tracked; inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) @@ -965,63 +1008,102 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, forget_all_cached_acls(inode); security_inode_invalidate_secctx(inode); if (offset >= 0) { + /* + * Everything below drives this inode's page cache on behalf + * of the revoke, so mark the task: writeback reached from + * here must not send a DLM request. See + * fuse_iomap_writeback_range(). + */ + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; + pg_start = offset >> PAGE_SHIFT; if (len <= 0) pg_end = -1; else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + + /* + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. + * + * A fenced writer may be waiting for a FUSE_READ or a + * FUSE_WRITE reply, so this waits on the server the same way + * the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. + */ + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); + /* - * A data invalidation means another (remote) entity is modifying - * the file. Two things happen here: + * A data invalidation means another (remote) entity is + * modifying the file. Two things happen here: * - * 1. Coherency. Drop the affected page-cache range so no local - * read returns a folio the remote modify has superseded. This - * runs under the write side of the per-inode coherency gate - * (wb_inval_rwsem), which fences cache-serving buffered reads - * and buffered writes out for the whole invalidate. Unlike the - * old best-effort trylock this BLOCKS -- the notify has - * priority: percpu_down_write() parks new gate readers, drains - * in-flight ones, then invalidates. A blocking writer here is - * safe only under a server that services request replies on - * threads other than the one delivering this notify: the write - * side waits for gate readers to drain, and a cache-miss read - * holds the read side across its FUSE_READ round-trip. redfs' - * dlm server provides that contract; a server that cannot must - * not enable writeback+dlm. + * 1. Coherency. Drop the affected page-cache range so no + * local read returns a folio the remote modify has + * superseded. Nothing is fenced out for it. A read + * racing the drop either misses and refetches or returns + * data that was current when it was copied. A write + * racing it is caught on the way out instead: its bytes + * were recorded before they were dirtied, this revoke + * marks the range rather than forgetting it, and + * writeback holds the range again before sending + * anything it finds marked that way. * - * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under - * fi->lock, updated for every data invalidation) of how fast - * these arrive; when they come in a rapid stream -- a remote - * writer repeatedly invalidating -- and the inode is also open - * for writing here, latch it into direct IO until the last - * writer closes or it is mmapped. When latched, drop the whole - * mapping rather than just the notified range, or dirty folios - * outside it would be invisible to the forced direct reads - * (stale read / lost write). Latching is opt-in via the - * enable_notify_dio module parameter and off by default; the - * average is kept up to date either way, so enabling it at - * runtime takes effect on the next storm rather than after a - * warm-up. Clearing it at runtime stops new latches but lets + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), + * under fi->lock, updated for every data invalidation) of + * how fast these arrive; when they come in a rapid stream + * -- a remote writer repeatedly invalidating -- and the + * inode is also open for writing here, latch it into + * direct IO until the last writer closes or it is mmapped. + * When latched, drop the whole mapping rather than just + * the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost + * write). Latching is opt-in via the enable_notify_dio + * module parameter and off by default; the average is kept + * up to date either way, so enabling it at runtime takes + * effect on the next storm rather than after a warm-up. + * Clearing it at runtime stops new latches but lets * already-latched inodes run out on the usual exits (last * writer closes, or mmap). * - * The gate (and the average) exist only for writeback+dlm regular - * files; elsewhere wb_sem is NULL and the invalidate runs - * unserialized (best-effort), as before. An mmapped inode - * keeps the gate -- fuse_cache_read_iter() and - * fuse_cache_write_iter() enter it unconditionally and rely - * on the revoke staying fenced -- but is never latched: - * a mapping needs the page cache, and fuse_file_mmap() - * reverts any latch it races with. + * The average and the latch exist only for writeback+dlm + * regular files; elsewhere there is no record to consult and + * the range is dropped as it always was. An mmapped inode is + * never latched: a mapping needs the page cache, and + * fuse_file_mmap() reverts any latch it races with. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi)) - wb_sem = fi->wb_inval_rwsem; + tracked = S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi); - if (wb_sem) { + if (tracked) { bool hot, has_writer, latched = false; + bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); @@ -1029,22 +1111,47 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, spin_unlock(&fi->lock); /* - * Priority write side: park new gate readers, - * drain in-flight ones, then invalidate. Blocks - * (unlike the old trylock) -- see the contract in - * the comment above. + * What this notify has to do. Nothing cached in the + * range means the drop is a no-op and the revoke is + * the whole job. Otherwise the page cache says + * whether the drop has to launder, which is what + * makes it wait for a FUSE_WRITE reply. */ - percpu_down_write(wb_sem); + has_pages = filemap_range_has_page(inode->i_mapping, + offset, end_byte); + may_be_dirty = filemap_range_needs_writeback( + inode->i_mapping, offset, end_byte); /* - * Revoke the DLM lock range under the gate write - * side, atomically with the page drop: gate readers - * re-validate their grant right after entering, and - * a grant that passed that check must stay visible - * for their whole gate hold. + * Put unwritten data on the server while the grant + * still covers it, rather than leaving it to the drop + * below. After the revoke writeback would have to + * take the range again to send those bytes: a DLM + * round trip from inside the handler the server is + * waiting on. do_writepages() runs in this context, + * so the grant is asked for before the revoke. + * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * + * Waited out here rather than left to the drop, which + * launders when the record says the range may be dirty + * and so waits for these same replies. One explicit + * wait, before the revoke, and the drop then finds + * nothing under writeback to block on. Either way a + * server that revokes from a thread it also needs to + * answer FUSE_WRITE on deadlocks here, the same + * contract fuse_notify_invalidate_range() states for a + * frozen inode. The error is left to the mapping, + * where fsync collects it. */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_revoke_inval_range(fi, offset, len); + if (has_pages && may_be_dirty) + filemap_write_and_wait_range(inode->i_mapping, + offset, end_byte); + + fuse_dlm_revoke_inval_range(fi, offset, len); if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && @@ -1060,28 +1167,35 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads). Otherwise just the - * notified range. + * the forced direct reads), and the record says + * nothing about the rest of the file, so launder. + * Otherwise just the notified range, and only if + * anything is cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1); - else + fuse_notify_invalidate_range(inode, 0, -1, true); + else if (has_pages) fuse_notify_invalidate_range(inode, pg_start, - pg_end); - - percpu_up_write(wb_sem); + pg_end, + may_be_dirty); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (DAX, backing, non-regular, - * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. */ + /* + * No record on this inode (DAX, backing, non-regular, + * or no DLM), so assume the range can hold unwritten + * data and drop it as before. + */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end, + true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); + fuse_notify_ctx_leave(notify_ctx); } iput(inode); return 0; @@ -1488,7 +1602,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, /* module option for now */ fc->compound_open_getattr = enable_compound; - xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1538,7 +1651,6 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); - xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } } @@ -1869,8 +1981,31 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, } if (flags & FUSE_ASYNC_DIO) fc->async_dio = 1; - if (flags & FUSE_WRITEBACK_CACHE) + if (flags & FUSE_WRITEBACK_CACHE) { + /* + * A buffered write goes through iomap, which + * tracks a folio a block at a time and fills + * any block the write covers only part of. + * Writeback then sends whole dirty blocks. + * Both of those are cut at the page in this + * filesystem: fuse_dlm_buffered_write() sends + * the unaligned edges of a write to the server + * itself so that no partly written block is + * ever dirtied, and it cuts at PAGE_SIZE. + * + * A block that is not a page breaks that, and + * quietly: the fill would come back for the + * remainder of an edge block, from a server + * that need not hold anything there. Refuse + * the connection instead. + */ + if (fc->blkbits != PAGE_SHIFT) { + pr_err("fuse: writeback cache needs a page sized block, got %u\n", + 1U << fc->blkbits); + ok = false; + } fc->writeback_cache = 1; + } if (flags & FUSE_PARALLEL_DIROPS) fc->parallel_dirops = 1; if (flags & FUSE_HANDLE_KILLPRIV)