From 7499e44c802a045737de4b48c4baaa0dc26fd4ea Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Wed, 26 Aug 2026 14:59:50 +0200 Subject: [PATCH 01/17] fuse: refuse a writeback connection whose block is not a page fuse_dlm_buffered_write() sends the unaligned edges of a write to the server itself so no partly written block is dirtied, and cuts at PAGE_SIZE. Nothing checked that a block is a page: fc->blkbits keeps whatever blksize= asked for, and attr->blksize sets inode->i_blkbits per inode. iomap then goes back for the remainder of an edge block and dirties what the write already sent. Refuse the connection at FUSE_INIT when a writeback cache is negotiated on a block that is not a page, and pin inode->i_blkbits afterwards. st_blksize still reports what the server named. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 31 +++++++++++++++++++------------ fs/fuse/inode.c | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 15 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 7c5b5e8b4ba268..f28a48ac59dcfb 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1805,18 +1805,25 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, } /* - * Buffered write under DLM. A partial page dirtied for writeback would - * have to be completed by reading the untouched remainder back from the - * server, and for a write past the server EOF that READ can only return - * zero bytes: a wasted round trip per unaligned edge. So cache only the - * page-aligned interior, whole pages need no read-modify-write, and - * route the unaligned head and tail straight through to the server. The - * writethrough path writes just those bytes and leaves the page - * non-uptodate, doing no read, and each edge lands as an independent - * FUSE_WRITE carrying FUSE_WRITE_CACHE like the writeback it replaces, - * so writers sharing a boundary page accumulate their bytes on the - * server. Aligned writes take the interior path whole; a - * sub-page write with no aligned interior goes fully through. + * Buffered write under DLM. A partly written page dirtied for + * writeback would have to be completed by reading the untouched + * remainder back from the server, and for a write past the server EOF + * that READ can only return zero bytes: a wasted round trip per + * unaligned edge. So cache only the page-aligned interior, whole pages + * need no read-modify-write, and route the unaligned head and tail + * straight through to the server. The writethrough path writes just + * those bytes and leaves the page non-uptodate, doing no read, and each + * edge lands as an independent FUSE_WRITE carrying FUSE_WRITE_CACHE + * like the writeback it replaces, so writers sharing a boundary page + * accumulate their bytes on the server. Aligned writes take the + * interior path whole; a sub-page write with no aligned interior goes + * fully through. + * + * The page is the block here: iomap tracks a folio a block at a time + * and __iomap_write_begin() skips the fill only for a block the write + * covers whole, so the cut has to land on block bounds. A writeback + * connection is refused unless the two are the same size; see + * process_init_reply(). */ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, struct iov_iter *from, diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index d54676b73abf9e..bb52782c064d3b 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -371,8 +371,17 @@ static void fuse_change_attributes_common_sx(struct inode *inode, } } - /* Common fields for both statx and getattr */ - if (attr->blksize != 0) + /* + * Common fields for both statx and getattr. + * + * A writeback connection was refused at FUSE_INIT unless its block + * is a page, for the reasons given there, and a server naming a + * different one per inode does not get to take that back. What it + * named is still reported as st_blksize out of + * fi->cached_i_blkbits; this is only what the page cache is + * tracked in. + */ + if (attr->blksize != 0 && !fc->writeback_cache) inode->i_blkbits = ilog2(attr->blksize); else inode->i_blkbits = inode->i_sb->s_blocksize_bits; @@ -1869,8 +1878,31 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, } if (flags & FUSE_ASYNC_DIO) fc->async_dio = 1; - if (flags & FUSE_WRITEBACK_CACHE) + if (flags & FUSE_WRITEBACK_CACHE) { + /* + * A buffered write goes through iomap, which + * tracks a folio a block at a time and fills + * any block the write covers only part of. + * Writeback then sends whole dirty blocks. + * Both of those are cut at the page in this + * filesystem: fuse_dlm_buffered_write() sends + * the unaligned edges of a write to the server + * itself so that no partly written block is + * ever dirtied, and it cuts at PAGE_SIZE. + * + * A block that is not a page breaks that, and + * quietly: the fill would come back for the + * remainder of an edge block, from a server + * that need not hold anything there. Refuse + * the connection instead. + */ + if (fc->blkbits != PAGE_SHIFT) { + pr_err("fuse: writeback cache needs a page sized block, got %u\n", + 1U << fc->blkbits); + ok = false; + } fc->writeback_cache = 1; + } if (flags & FUSE_PARALLEL_DIROPS) fc->parallel_dirops = 1; if (flags & FUSE_HANDLE_KILLPRIV) From 8c2ff0e081ec2970dfc6a0bcdb15f66f56d83eab Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Fri, 21 Aug 2026 14:07:44 +0200 Subject: [PATCH 02/17] fuse: take a DLM read lock for the readahead window Readahead fills the page cache past the range fuse_cache_read_iter() locked, so those folios get no revoke when a remote node writes them. Request a read grant over the whole window in fuse_readahead() before any folio is consumed, and skip the window when the request fails. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index f28a48ac59dcfb..e1ce3c0487da33 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1154,6 +1154,34 @@ static void fuse_readahead(struct readahead_control *rac) if (fuse_is_bad(inode)) return; + /* + * Readahead fills the page cache past the range the reader locked, + * so take a DLM read grant over the whole window here too. Folios + * the server handed out no lock for are folios it will not revoke + * when a remote node writes them, and a later read would be served + * from stale cache. Take the grant before any folio is pulled off + * @rac, so the window is either fully covered or not populated. + * + * Speculative pages are not worth serving uncovered: on a failed + * request drop the window and let read_pages() clean up the folios + * left in @rac. A server without DLM support answers -ENOSYS and + * clears fc->dlm, which is not a failure. + * + * This can run inside the coherency gate, which + * fuse_cache_read_iter() holds across generic_file_read_iter(), so + * the round trip leans on the same server contract that lets a + * cache-miss FUSE_READ block there: replies are serviced on threads + * other than the one delivering a NOTIFY invalidate. + */ + if (fc->writeback_cache && fc->dlm) { + int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), + readahead_length(rac), + FUSE_PAGE_LOCK_READ); + + if (err < 0 && err != -ENOSYS) + return; + } + max_pages = min_t(unsigned int, fc->max_pages, fc->max_read / PAGE_SIZE); From 6bc763732650375f7db59f45d4c2639ed048fdb3 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 1 Sep 2026 14:05:28 +0200 Subject: [PATCH 03/17] fuse: hold DLM grants as a bitmap and drive every path from it The record was an interval tree of ranges carrying a lock mode and a revoke generation, walked under one lock per inode. Splitting and merging ranges on every grant made it the contended structure on a file several threads write. Record coverage as two bitmaps per aligned file region instead, a bit per page for granted and for granted-for-write. A region is a fixed span, so recording a grant neither allocates nor rearranges anything, and the regions live in an xarray, so threads writing far apart never touch the same one. A grant still on the wire stays out of the bitmaps and waits on a pending list where a revoke marks it, which keeps a reply that crosses a revoke from recording coverage the server took back. Adapt the paths to it: reads and readahead take a read grant, a cached write takes the write grant for the range it dirties and sends its unaligned edges straight through, writeback confirms the grant for every run it sends, and a NOTIFY invalidate drops the record for the range it revokes before it drops the page cache. Signed-off-by: Horst Birthelmer --- fs/fuse/dir.c | 42 +- fs/fuse/file.c | 627 +++++++++++++++--------- fs/fuse/fuse_dlm_cache.c | 999 +++++++++++++++++++-------------------- fs/fuse/fuse_dlm_cache.h | 137 +++++- fs/fuse/fuse_i.h | 15 - fs/fuse/inode.c | 246 +++++----- 6 files changed, 1166 insertions(+), 900 deletions(-) diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 5f426228f4c76a..97f7cdcab43289 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -726,6 +726,10 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, memset(&inarg, 0, sizeof(inarg)); memset(&outentry, 0, sizeof(outentry)); inarg.flags = flags; + + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; inarg.mode = mode; inarg.umask = current_umask(); @@ -2106,34 +2110,24 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, WARN_ON(!(attr->ia_valid & ATTR_SIZE)); WARN_ON(attr->ia_size != 0); if (fc->atomic_o_trunc) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* * No need to send request to userspace, since actual * truncation has already been done by OPEN. But still * need to truncate page cache. * - * Revoke and drop under the coherency gate write side, - * like the NOTIFY invalidate path: a gate reader that - * already re-validated its grant must not have the - * lock tree and the cache yanked mid-hold, or it - * would repopulate the truncated range trusting a - * grant that no longer exists. Waiting for gate - * readers here is safe: we hold i_rwsem exclusive, so - * no gate holder can be waiting on it (the write path - * takes i_rwsem before the gate, the read path never - * takes it). + * Dropping every grant here does not need a reader or + * writer fenced out: truncate_pagecache() discards the + * folios rather than writing them, and a write racing + * this is a write racing an O_TRUNC open, which has no + * order to preserve. i_rwsem is held exclusive + * anyway, so no cached write is in progress. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) fuse_dlm_cache_release_locks(fi); spin_lock(&fi->lock); i_size_write(inode, 0); spin_unlock(&fi->lock); truncate_pagecache(inode, 0); - if (wb_sem) - percpu_up_write(wb_sem); goto out; } file = NULL; @@ -2237,23 +2231,17 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, */ if ((is_truncate || !is_wb) && S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; - /* - * Revoke and drop under the coherency gate write side; see - * the atomic-O_TRUNC branch above. i_rwsem is held - * exclusive here as well (setattr), so waiting out gate - * readers cannot deadlock. + * Revoke past the new size and drop what is beyond it; see + * the atomic-O_TRUNC branch above for why this needs nothing + * fenced out. i_rwsem is held exclusive here as well. */ - if (wb_sem) - percpu_down_write(wb_sem); if (fc->dlm && fc->writeback_cache) - fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, -1); + fuse_dlm_unlock_range(fi, outarg.attr.size & PAGE_MASK, + U64_MAX); truncate_pagecache(inode, outarg.attr.size); invalidate_inode_pages2(mapping); - if (wb_sem) - percpu_up_write(wb_sem); } clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e1ce3c0487da33..e21a08dd5704df 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -71,6 +71,17 @@ static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) inarg.flags &= ~O_TRUNC; + /* + * With the writeback cache the kernel owns append positioning: + * writeback sends FUSE_WRITE with explicit offsets, and a server + * that opens its backing file O_APPEND has pwrite(2) ignore them + * (Linux appends regardless of offset). Any re-sent or reordered + * run is then placed at EOF: duplicated data and a growing file. + * Do not hand the flag to the server at all. + */ + if (fm->fc->writeback_cache) + inarg.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -174,6 +185,10 @@ static int fuse_compound_open_getattr(struct fuse_mount *fm, u64 nodeid, if (!fm->fc->atomic_o_trunc) open_in.flags &= ~O_TRUNC; + /* The kernel owns append positioning; see fuse_send_open() */ + if (fm->fc->writeback_cache) + open_in.flags &= ~O_APPEND; + if (fm->fc->handle_killpriv_v2 && (open_in.flags & O_TRUNC) && !capable(CAP_FSETID)) open_in.open_flags |= FUSE_OPEN_KILL_SUIDGID; @@ -384,10 +399,20 @@ static int fuse_open(struct inode *inode, struct file *file) if (is_wb_truncate || dax_truncate) fuse_release_nowrite(inode); if (!err) { - if (is_truncate) + if (is_truncate) { + /* + * Every grant goes with the cache, as on the + * fuse_do_setattr() O_TRUNC path: a record left + * behind would keep naming bytes the folios no + * longer hold. i_rwsem is held exclusive + * (is_wb_truncate), so no cached write is mid-record. + */ + if (fc->dlm && fc->writeback_cache) + fuse_dlm_cache_release_locks(fi); truncate_pagecache(inode, 0); - else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) { invalidate_inode_pages2(inode->i_mapping); + } } if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); @@ -469,8 +494,8 @@ void fuse_file_release(struct inode *inode, struct fuse_file *ff, * If this release dropped the last writer, fuse_prepare_release() * cleared the forced-direct-IO latch (under fi->lock). Drop any clean * folios a read racing the latch may have repopulated so they cannot be - * served stale once caching mode resumes. No inode lock or - * wb_inval_rwsem: release may run on the fuse server thread (async fput + * served stale once caching mode resumes. No inode lock: release may + * run on the fuse server thread (async fput * from aio completion), where blocking on a contended inode lock could * stall the connection. Writes were routed direct while latched, so * only clean folios exist and this invalidate is server-free; the last @@ -556,11 +581,15 @@ u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) return (u64) v0 + ((u64) v1 << 32); } +struct fuse_wb_token; + struct fuse_writepage_args { struct fuse_io_args ia; struct list_head queue_entry; struct inode *inode; struct fuse_sync_bucket *bucket; + /* One per entry of ia.ap.folios, see struct fuse_wb_token */ + struct fuse_wb_token **tokens; }; /* @@ -1007,16 +1036,137 @@ static int fuse_do_readfolio(struct file *file, struct folio *folio, return 0; } +/** + * fuse_read_folio_range - read part of a folio from the server + * @file: file to read through + * @folio: the folio to fill + * @off: offset within @folio to start at + * @len: bytes to read + * + * fuse_do_readfolio() cannot serve a partial folio: it asks for + * page_zeroing, and fuse_copy_folio() answers that by zeroing the whole + * folio whenever the request covers less than all of it. Ask without it + * and zero exactly what the reply left short, which is the server saying + * the file ends there. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_range(struct file *file, struct folio *folio, + size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = folio_pos(folio) + off; + struct fuse_folio_desc desc = { + .offset = off, + .length = len, + }; + struct fuse_io_args ia = { + .ap.args.out_pages = true, + .ap.num_folios = 1, + .ap.folios = &folio, + .ap.descs = &desc, + }; + ssize_t res; + + /* Don't overflow end offset */ + if (pos + (desc.length - 1) == LLONG_MAX) + desc.length--; + + fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) { + /* See fuse_do_readfolio() for why READ can return -EDEADLK */ + if ((res == -EDEADLK || res == -EAGAIN) && fm->fc->dlm) + res = AOP_TRUNCATED_PAGE; + return res; + } + + if (res < desc.length) + folio_zero_range(folio, off + res, desc.length - res); + + return 0; +} + +/** + * fuse_read_folio_merge - fill @folio without disturbing what it holds + * @file: file to read through + * @folio: the folio to fill + * + * iomap tracks a folio a block at a time, and a write that covered some + * of its blocks and not others leaves it valid in the ones it covered + * and not in the rest. The valid ones hold what that write put there, + * which writeback may not have sent yet; reading over them would lose + * it. Fetch the rest, in as few requests as the gaps allow. + * + * A folio the page cache tracks in one piece has no per block state to + * ask, and none to have: it is dirty only if a write covered it whole, + * and then it is valid and never reaches here. + * + * Return: 0, AOP_TRUNCATED_PAGE, or a negative error. + */ +static int fuse_read_folio_merge(struct file *file, struct folio *folio) +{ + size_t bsize = i_blocksize(folio->mapping->host); + size_t size = folio_size(folio); + size_t off = 0; + + while (off < size) { + size_t run = 0; + int err; + + /* Skip what the folio already holds */ + while (off < size && + iomap_is_partially_uptodate(folio, off, bsize)) + off += bsize; + + /* Take the gap behind it in one request */ + while (off + run < size && + !iomap_is_partially_uptodate(folio, off + run, bsize)) + run += bsize; + + if (!run) + break; + + err = fuse_read_folio_range(file, folio, off, run); + if (err) + return err; + off += run; + } + + return 0; +} + static int fuse_read_folio(struct file *file, struct folio *folio) { struct inode *inode = folio->mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); int err; err = -EIO; if (fuse_is_bad(inode)) goto out; - err = fuse_do_readfolio(file, folio, 0, folio_size(folio)); + /* + * Writeback unlocks a folio as soon as it has handed it over, with + * the writeback flag still on it, so this can be reached while a + * FUSE_WRITE is still reading out of it. Filling it now would + * rewrite what is being sent, and past the end of the file the reply + * comes back short and zeroes it. Nothing reached here before a + * partial write started leaving folios invalid, because a dirty + * folio was always valid and never came this way. + */ + folio_wait_writeback(folio); + + /* + * Only a folio still holding what a write put in it has anything to + * keep. One that was merely being written back is clean by now, + * waited out just above, and reads whole. + */ + if (fc->writeback_cache && folio_test_dirty(folio)) + err = fuse_read_folio_merge(file, folio); + else + err = fuse_do_readfolio(file, folio, 0, folio_size(folio)); if (!err) folio_mark_uptodate(folio); @@ -1036,7 +1186,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, size_t off = offset_in_folio(folio, pos); int ret; - ret = fuse_do_readfolio(file, folio, off, len); + ret = fuse_read_folio_range(file, folio, off, len); /* * TEMPORARY WORKAROUND for iomap write deadlock: @@ -1167,11 +1317,9 @@ static void fuse_readahead(struct readahead_control *rac) * left in @rac. A server without DLM support answers -ENOSYS and * clears fc->dlm, which is not a failure. * - * This can run inside the coherency gate, which - * fuse_cache_read_iter() holds across generic_file_read_iter(), so - * the round trip leans on the same server contract that lets a - * cache-miss FUSE_READ block there: replies are serviced on threads - * other than the one delivering a NOTIFY invalidate. + * The round trip is taken before any folio of the window is locked + * and with nothing fenced out, so it holds up this reader and + * nothing else. */ if (fc->writeback_cache && fc->dlm) { int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), @@ -1255,21 +1403,12 @@ static void fuse_readahead(struct readahead_control *rac) static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to); -/* - * Bound on re-requesting a revoked DLM grant before a cached read is - * served unlocked; see fuse_cache_read_iter(). - */ -#define FUSE_DLM_READ_RETRIES 3 - static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) { struct file *file = iocb->ki_filp; struct inode *inode = file->f_mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; ssize_t res; - int lock_err = 0; /* * In auto invalidate mode, always update attributes on read. @@ -1287,65 +1426,35 @@ static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) /* if we have dlm support acquire a read lock for the area * we are reading from. */ if (fc->writeback_cache && fc->dlm) - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); + fuse_get_dlm_lock(file, iocb->ki_pos, iov_iter_count(to), + FUSE_PAGE_LOCK_READ); /* - * Fence the cache-serving read against a NOTIFY invalidate so we never - * hand back a folio the server has just superseded. The gate read side - * is per-CPU cheap; the NOTIFY holds the write side with priority. - * Re-check the forced-DIO latch under it: if a storm latched us while we - * waited on a pending writer, reroute to direct like the buffered write - * path, so we do not repopulate the cache the latch just dropped. - * wb_sem is NULL on non-writeback+dlm mounts (gate inactive). + * A NOTIFY invalidate racing this read drops the folios it + * supersedes, so the read either misses and refetches or returns + * data that was current when it was copied. There is nothing to + * fence: unlike a write, a read leaves nothing behind that could + * reach the server under a grant it no longer holds. */ - if (wb_sem) { - int tries = FUSE_DLM_READ_RETRIES; + if (fuse_inode_force_dio(inode)) { + size_t count = iov_iter_count(to); -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - return fuse_direct_read_iter(iocb, to); - } /* - * The DLM lock was requested before entering the gate, and - * the NOTIFY invalidate we may just have waited on revokes - * locks under the gate write side. Re-check the grant here - * and re-request with the gate dropped, so a - * FUSE_DLM_WB_LOCK round trip never parks a pending - * invalidate behind our own gate hold. Once the check - * passes the lock cannot go away for the rest of the gate - * hold. A failed or unrecorded request falls through - * unlocked, as before: the retry is taken even then (the - * latch must be re-checked under the re-entered gate), so - * lock_err has to stay sticky across it -- seeded by the - * pre-gate request above -- or a grant that failed would - * be re-requested forever. The retry is also bounded: a - * remote writer can revoke each successful grant before - * the gate is re-entered, and a reader-only inode has no - * force-DIO latch to end such a storm, so after - * FUSE_DLM_READ_RETRIES re-requests the read is served - * unlocked rather than looping without bound. + * A write that passed this same check just before the latch + * took hold dirtied the page cache after the notify dropped + * it, and a direct read does not look there. Send it first. */ - if (!lock_err && fc->dlm && tries-- > 0 && - !fuse_dlm_lock_is_held(fi, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ)) { - percpu_up_read(wb_sem); - lock_err = fuse_get_dlm_lock(file, iocb->ki_pos, - iov_iter_count(to), - FUSE_PAGE_LOCK_READ); - goto retry; + if (count) { + res = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + if (res) + return res; } + return fuse_direct_read_iter(iocb, to); } res = generic_file_read_iter(iocb, to); - if (wb_sem) - percpu_up_read(wb_sem); - return res; } @@ -1861,32 +1970,37 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, loff_t end = pos + iov_iter_count(from); loff_t mid_start = round_up(pos, PAGE_SIZE); loff_t mid_end = round_down(end, PAGE_SIZE); + /* Unaligned head, cached interior of whole pages, unaligned tail */ + const struct { + loff_t len; + bool through; + } chunk[] = { + { mid_start - pos, true }, + { mid_end - mid_start, false }, + { end - mid_end, true }, + }; ssize_t res, total = 0; + unsigned int i; /* No whole page inside the write: nothing cacheable, all through. */ if (mid_end <= mid_start) return fuse_perform_write(iocb, from, true); - /* Unaligned head [pos, mid_start): through. */ - res = fuse_dlm_write_chunk(iocb, from, file, mid_start - pos, true); - if (res < 0) - return res; - total += res; - if (res < mid_start - pos) - return total; - - /* Aligned interior [mid_start, mid_end): cached whole pages. */ - res = fuse_dlm_write_chunk(iocb, from, file, mid_end - mid_start, false); - if (res < 0) - return total; - total += res; - if (res < mid_end - mid_start) - return total; - - /* Unaligned tail [mid_end, end): through. */ - res = fuse_dlm_write_chunk(iocb, from, file, end - mid_end, true); - if (res > 0) + /* + * Every chunk reports a failure the same way: the error while + * nothing has landed, a short write once something has. Returning + * what has landed when that is nothing reports no error and no + * bytes, which fuse_cache_write_iter() turns into the full count. + */ + for (i = 0; i < ARRAY_SIZE(chunk); i++) { + res = fuse_dlm_write_chunk(iocb, from, file, chunk[i].len, + chunk[i].through); + if (res < 0) + return total ? total : res; total += res; + if (res < chunk[i].len) + break; + } return total; } @@ -1937,19 +2051,13 @@ static void fuse_cache_wr_unlock(struct inode *inode, bool exclusive) * fc->dlm: the server has no DLM, proceed as a plain cached write. Any * other failure means the cache would be dirtied without DLM coverage - * the caller must fail the write instead. A granted-but-unrecorded - * lock (positive return) is covered cluster-wide; proceed, but flag it - * so the in-gate re-validation skips a check an invisible grant could - * never pass. + * lock (positive return) is covered cluster-wide; proceed. */ -static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len, - bool *unrecorded) +static int fuse_cache_wr_dlm_lock(struct file *file, loff_t pos, size_t len) { int err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); - if (err < 0 && err != -ENOSYS) - return err; - *unrecorded = err > 0; - return 0; + return (err < 0 && err != -ENOSYS) ? err : 0; } static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) @@ -1962,13 +2070,8 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) ssize_t err, count; struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); - struct percpu_rw_semaphore *wb_sem = fi->wb_inval_rwsem; bool writeback = false; - bool wb_guard = false; - bool exclusive = true; - bool dlm_unrecorded = false; - loff_t dlm_pos = 0; - size_t dlm_len = 0; + bool exclusive; if (fuse_inode_force_dio(inode)) return fuse_direct_write_iter(iocb, from); @@ -2012,98 +2115,64 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) writeback = true; } - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); - /* * Request the DLM write lock before taking i_rwsem: the request is * an unbounded cluster round trip, and holding the writer-priority * rwsem across it would park a truncate -- and behind it every - * later writer -- for the duration. The grant-to-use window this - * leaves open is closed by the in-gate re-validation below. Only - * the append case must wait for the lock: its range depends on - * i_size, which is stable only under the exclusive inode lock. + * later writer -- for the duration. Only the append case must wait + * for the lock: its range depends on i_size, which is settled by + * generic_write_checks() under the exclusive inode lock. + * + * The request may find that the server has no DLM at all and clear + * fc->dlm, so pick the lock mode after it rather than before. The + * relaxed shared lock is only sound under DLM: the shared path + * claims the i_size extension up front, which stops iomap from + * zeroing beyond EOF, and the zero-fill that replaces it in + * fuse_iomap_read_folio_range() is itself gated on fc->dlm. Chosen + * too early, an expanding write would fall through to a READ of a + * range that cannot hold data -- which fails outright on a handle + * the client opened write-only. */ if (writeback && fc->dlm && !(iocb->ki_flags & IOCB_APPEND)) { - dlm_pos = iocb->ki_pos; - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, + iov_iter_count(from)); if (err) return err; - - /* - * The request above may have found that the server has no DLM - * at all, in which case it cleared fc->dlm. The relaxed shared - * lock was chosen just before, while fc->dlm still read 1, and - * it is only sound under DLM: the shared path claims the i_size - * extension up front, which stops iomap from zeroing beyond - * EOF, and the zero-fill that replaces it in - * fuse_iomap_read_folio_range() is itself gated on fc->dlm. - * Left as chosen, an expanding write would fall through to a - * READ of a range that cannot hold data -- which fails outright - * on a handle the client opened write-only. Re-decide now, - * while no lock is held yet. - */ - exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); } + exclusive = fuse_cache_wr_exclusive_lock(iocb, writeback); + if (exclusive) inode_lock(inode); else inode_lock_shared(inode); - /* note that this small code dup will save us a lot of headache later - * when appends are done concurrently without using parallel direct writes */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { - /* - * An append write lands at the current EOF no matter what - * ki_pos holds: generic_write_checks() rewrites ki_pos to - * i_size for IOCB_APPEND. Lock where the data will land. - */ - dlm_pos = i_size_read(inode); - dlm_len = iov_iter_count(from); - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) - goto out; - } - err = count = generic_write_checks(iocb, from); if (err <= 0) goto out; /* - * The exclusive inode lock does not pin i_size for the append: - * attribute replies move it under fi->lock alone, so - * generic_write_checks() may have put ki_pos past the granted - * range. Re-lock where the write really lands; dlm_pos tracks it - * so the in-gate re-validation below guards the same range. + * An append lands at the EOF generic_write_checks() has just written + * into ki_pos, not where the caller pointed, and the exclusive inode + * lock does not pin i_size either: attribute replies move it under + * fi->lock alone. Take the grant here, where the range is settled. */ - if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND) && - iocb->ki_pos != dlm_pos) { - dlm_pos = iocb->ki_pos; - dlm_len = count; - - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); + if (writeback && fc->dlm && (iocb->ki_flags & IOCB_APPEND)) { + err = fuse_cache_wr_dlm_lock(file, iocb->ki_pos, count); if (err) goto out; } /* - * Kill suid/sgid and stamp the timestamps here, before the gate, - * instead of leaving them next to the write itself. kiocb_modified() - * -> file_remove_privs() is the one that reaches the server: without - * handle_killpriv[_v2] fuse_setattr() kills the bits by asking it (a - * FUSE_GETATTR to refresh the mode, then a FUSE_SETATTR, which for a - * writeback inode first flushes and freezes writepages), and - * security_inode_killpriv() can drop the capability xattr with another - * round trip. A server may have to invalidate this inode from inside - * such a handler; its NOTIFY_INVAL_INODE then blocks in - * percpu_down_write() draining a gate reader that is itself waiting for - * the reply. Nothing held under the gate may wait for the server. + * Kill suid/sgid and stamp the timestamps here, ahead of the write + * itself. kiocb_modified() -> file_remove_privs() is the one that + * reaches the server: without handle_killpriv[_v2] fuse_setattr() + * kills the bits by asking it (a FUSE_GETATTR to refresh the mode, + * then a FUSE_SETATTR, which for a writeback inode first flushes and + * freezes writepages), and security_inode_killpriv() can drop the + * capability xattr with another round trip. A server may have to + * invalidate this inode from inside such a handler, and it must not + * find this write holding anything it needs. * * This also runs before the forced-DIO re-route below, so a re-routed * write repeats it; there is nothing left to do the second time. @@ -2112,31 +2181,32 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err) goto out; - wb_guard = !!wb_sem; - if (wb_guard) { -retry: - percpu_down_read(wb_sem); - if (fuse_inode_force_dio(inode)) { - percpu_up_read(wb_sem); - fuse_cache_wr_unlock(inode, exclusive); - return fuse_direct_write_iter(iocb, from); - } - if (writeback && fc->dlm && !dlm_unrecorded && - !fuse_dlm_lock_is_held(fi, dlm_pos, dlm_len, - FUSE_PAGE_LOCK_WRITE)) { - percpu_up_read(wb_sem); - err = fuse_cache_wr_dlm_lock(file, dlm_pos, dlm_len, - &dlm_unrecorded); - if (err) { - /* The gate is already dropped; funnel the - * failure through the one audited exit. */ - wb_guard = false; - goto out; - } - goto retry; - } + if (fuse_inode_force_dio(inode)) { + /* + * As on the read side, only worse: the direct write would + * land under whatever a write racing the latch left dirty, + * and the invalidate fuse_direct_write_iter() does after it + * launders rather than drops, putting that folio on the + * server on top. Send it first and the order is ordinary. + */ + if (count) + err = filemap_write_and_wait_range(inode->i_mapping, + iocb->ki_pos, iocb->ki_pos + count - 1); + fuse_cache_wr_unlock(inode, exclusive); + if (err) + return err; + return fuse_direct_write_iter(iocb, from); } + /* + * A NOTIFY invalidate can revoke the grant requested above between + * here and the dirtying below, and nothing stops it: the bytes are + * caught on the way out instead. fuse_dlm_unlock_range() keeps a + * revoked range for as long as there is page cache under it, and + * writeback holds the range again before sending anything. So a + * write racing a revoke costs a round trip, not coverage. + */ + task_io_account_write(count); if (iocb->ki_flags & IOCB_DIRECT) { @@ -2181,7 +2251,13 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) } spin_unlock(&fi->lock); - /* Zero the tail of the folio straddling the old EOF. */ + /* + * Zero the tail of the folio straddling the old EOF. + * Inert while the fuse block size is PAGE_SIZE, which + * it always is, and nothing is recorded for it either + * way: claiming the whole gap as written would hand + * writeback bytes no one wrote. + */ if (extended && orig_size < pos) pagecache_isize_extended(inode, orig_size, pos); } @@ -2189,7 +2265,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) /* * Under DLM the unaligned edges go through to the server * instead of being completed by a read-modify-write READ - * (see fuse_dlm_buffered_write()); only whole pages are + * (see fuse_dlm_buffered_write()); only whole blocks are * cached for writeback. */ if (fc->dlm) @@ -2224,8 +2300,6 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) written = fuse_perform_write(iocb, from, false); } out: - if (wb_guard) - percpu_up_read(wb_sem); fuse_cache_wr_unlock(inode, exclusive); if (written > 0) written = generic_write_sync(iocb, written); @@ -2584,6 +2658,77 @@ static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, return iter_file_splice_write(pipe, out, ppos, len, flags); } +/* + * A folio is written back one recorded run at a time, and a run that does + * not reach both folio edges forces a new request, so the runs of one folio + * end up in requests that complete independently. iomap counts a folio's + * outstanding writes in ifs->write_bytes_pending, but a folio of a single + * block carries no iomap_folio_state, and there iomap_finish_folio_write() + * ends the writeback on every call. Count the runs here instead and end + * the folio writeback once, on the last one. + */ +struct fuse_wb_token { + refcount_t refs; + struct inode *inode; + struct folio *folio; +}; + +/* + * Take @folio into writeback and open the count. The caller keeps the + * returned reference as a bias, so the count cannot reach zero while + * further runs of the same folio are still being queued. + */ +static struct fuse_wb_token *fuse_wb_token_alloc(struct inode *inode, + struct folio *folio) +{ + struct fuse_wb_token *token; + + /* As iomap allocates the state this stands in for */ + token = kmalloc(sizeof(*token), GFP_NOFS | __GFP_NOFAIL); + refcount_set(&token->refs, 1); + token->inode = inode; + token->folio = folio; + iomap_start_folio_write(inode, folio, 1); + + return token; +} + +static struct fuse_wb_token *fuse_wb_token_get(struct fuse_wb_token *token) +{ + refcount_inc(&token->refs); + return token; +} + +static void fuse_wb_token_put(struct fuse_wb_token *token) +{ + if (token && refcount_dec_and_test(&token->refs)) { + iomap_finish_folio_write(token->inode, token->folio, 1); + kfree(token); + } +} + +/* + * The folios, descs and tokens of a writeback request come from one + * allocation, which kfree(ap->folios) releases. + */ +static struct folio **fuse_wb_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **descs, + struct fuse_wb_token ***tokens) +{ + struct folio **folios; + + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc) + + sizeof(struct fuse_wb_token *)), flags); + if (!folios) + return NULL; + + *descs = (void *) (folios + nfolios); + *tokens = (void *) (*descs + nfolios); + + return folios; +} + static void fuse_writepage_free(struct fuse_writepage_args *wpa) { struct fuse_args_pages *ap = &wpa->ia.ap; @@ -2610,7 +2755,7 @@ static void fuse_writepage_finish(struct fuse_writepage_args *wpa) * scope of the fi->lock alleviates xarray lock * contention and noticeably improves performance. */ - iomap_finish_folio_write(inode, ap->folios[i], 1); + fuse_wb_token_put(wpa->tokens[i]); wake_up(&fi->page_waitq); } @@ -2758,7 +2903,8 @@ static struct fuse_writepage_args *fuse_writepage_args_alloc(void) if (wpa) { ap = &wpa->ia.ap; ap->num_folios = 0; - ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + ap->folios = fuse_wb_folios_alloc(1, GFP_NOFS, &ap->descs, + &wpa->tokens); if (!ap->folios) { kfree(wpa); wpa = NULL; @@ -2783,13 +2929,15 @@ static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, } static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, - uint32_t folio_index, loff_t offset, unsigned len) + uint32_t folio_index, loff_t offset, unsigned int len, + struct fuse_wb_token *token) { struct fuse_args_pages *ap = &wpa->ia.ap; ap->folios[folio_index] = folio; ap->descs[folio_index].offset = offset; ap->descs[folio_index].length = len; + wpa->tokens[folio_index] = fuse_wb_token_get(token); } static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, @@ -2822,6 +2970,12 @@ struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; unsigned int max_folios; + /* + * The folio currently being split into runs, and the count that + * holds its writeback open until the last run has been queued. + */ + struct folio *wb_folio; + struct fuse_wb_token *wb_token; /* * nr_bytes won't overflow since fuse_writepage_need_send() caps * wb requests to never exceed fc->max_pages (which has an upper bound @@ -2836,21 +2990,25 @@ static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, struct fuse_args_pages *ap = &data->wpa->ia.ap; struct folio **folios; struct fuse_folio_desc *descs; + struct fuse_wb_token **tokens; unsigned int nfolios = min_t(unsigned int, max_t(unsigned int, data->max_folios * 2, FUSE_DEFAULT_MAX_PAGES_PER_REQ), max_pages); WARN_ON(nfolios <= data->max_folios); - folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + folios = fuse_wb_folios_alloc(nfolios, GFP_NOFS, &descs, &tokens); if (!folios) return false; memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); + memcpy(tokens, data->wpa->tokens, + sizeof(struct fuse_wb_token *) * ap->num_folios); kfree(ap->folios); ap->folios = folios; ap->descs = descs; + data->wpa->tokens = tokens; data->max_folios = nfolios; return true; @@ -2932,7 +3090,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct inode *inode = wpc->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); - loff_t offset = offset_in_folio(folio, pos); + loff_t offset; WARN_ON_ONCE(!data); @@ -2942,6 +3100,39 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, return -EIO; } + /* + * A folio iomap has not asked about before: the one before it has all + * of its runs queued, so let go of the bias holding its count open. + */ + if (data->wb_folio != folio) { + fuse_wb_token_put(data->wb_token); + data->wb_token = NULL; + data->wb_folio = folio; + } + + /* + * Every run iomap reports dirty was written whole by this client: + * the unaligned edges of a cached write go to the server directly + * and the interior covers whole blocks, so nothing partly written + * is ever dirtied. There is nothing to classify, only the grant to + * make sure of: a revoke may have arrived since the write, and + * these bytes must not go out from under one. + * + * fuse_dlm_regrant_range() takes the range back when it has gone, + * and walks the record once under the lock held for read when it + * has not. A failure leaves the folio dirty, so the next writeback + * tries again; only a hard error stops it. + */ + if (fc->dlm && fc->writeback_cache) { + int err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); + + if (err < 0 && err != -ENOSYS) + return err; + } + + offset = offset_in_folio(folio, pos); + if (wpa && fuse_writepage_need_send(fc, pos, len, ap, data, wpc->wbc)) { fuse_writepages_send(inode, data); data->wpa = NULL; @@ -2957,9 +3148,16 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, ap = &wpa->ia.ap; } - iomap_start_folio_write(inode, folio, 1); + /* + * The first run of this folio that is actually sent takes it into + * writeback. A folio with no run at all never gets here, and iomap + * ends its writeback itself. + */ + if (!data->wb_token) + data->wb_token = fuse_wb_token_alloc(inode, folio); + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, - offset, len); + offset, len, data->wb_token); data->nr_bytes += len; ap->num_folios++; @@ -2976,6 +3174,11 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, WARN_ON_ONCE(!data); + /* No more runs are coming for the folio last seen */ + fuse_wb_token_put(data->wb_token); + data->wb_token = NULL; + data->wb_folio = NULL; + if (data->wpa) { WARN_ON(!data->wpa->ia.ap.num_folios); fuse_writepages_send(wpc->inode, data); @@ -3100,6 +3303,7 @@ static int fuse_get_page_mkwrite_lock(struct file *file, loff_t offset, size_t l fuse_abort_conn(fc); err = -EINVAL; } + return err; } /* @@ -3175,7 +3379,7 @@ static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) /* * If the inode was latched into forced direct IO after a remote-modify * notification, a mapping needs the page cache, so revert to caching - * mode. Revert without the inode lock or wb_inval_rwsem: ->mmap runs + * mode. Revert without the inode lock: ->mmap runs * under mmap_lock and the buffered write path holds both across a fault * on the user buffer (which takes mmap_lock), so taking either here * would invert lock order (ABBA). Clearing the latch and dropping the @@ -4019,23 +4223,6 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) fi->iocachectr = 0; init_waitqueue_head(&fi->page_waitq); init_waitqueue_head(&fi->direct_io_waitq); - /* - * Coherency gate for the forced-direct-IO feature; only writeback+dlm - * regular files need it. A percpu_rw_semaphore embeds per-CPU state, - * so allocate it out of line and only when the mount can use it rather - * than paying it on every inode. On failure leave it NULL: the gate - * stays inactive (best-effort invalidate) and the inode is still usable. - */ - fi->wb_inval_rwsem = NULL; - if (fc->writeback_cache && fc->dlm) { - struct percpu_rw_semaphore *sem = kmalloc(sizeof(*sem), GFP_KERNEL); - - if (sem && percpu_init_rwsem(sem)) { - kfree(sem); - sem = NULL; - } - fi->wb_inval_rwsem = sem; - } fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index bc6dbae2d5aeb0..b2a47fda316b40 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -1,596 +1,521 @@ // SPDX-License-Identifier: GPL-2.0-only /* * FUSE page lock cache implementation + * + * The shards record the grants the server has given this client, a bit + * per page. A grant still on the wire covers nothing and must not + * appear there, but a revoke has to be able to find it: otherwise a + * revoke processed before the grant is recorded removes nothing, and the + * grant recorded afterwards is never taken back. A request in flight + * therefore waits on cache->pending, where a revoke marks it killed and + * fuse_dlm_request_commit() drops the grant instead of recording it. + * + * Keeping requests out of the shards leaves every walker looking at + * grants alone. + * + * The record says nothing about the page cache under a page. What is + * cached there, and whether the server has seen it, is what the page + * cache itself answers. A revoked grant is therefore forgotten, not + * kept: writeback holds the range again for every run it sends, and an + * absent record and a revoked one both make it ask. */ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include #include +#include #include #include -#include +#include +#include + + +/* + * How often to ask again for a grant a revoke killed while it was in + * flight, before giving up on the range. Each pass is a round trip. + */ +#define FUSE_DLM_GRANT_RETRIES 16 +/* + * How far beyond the requested range a grant is recorded. + * + * A server may grant more than was asked for, and recording the extra is + * what lets the writes that follow skip the round trip entirely. But + * coverage is kept per shard, so the number of records a grant creates + * grows with its size, and outarg is the server's to choose: an + * unbounded grant would be an unbounded amount of work here. Cap it. + * Recording less than the server gave is safe - it only costs a + * re-request - and a cap this size still covers thousands of writes. + */ +#define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) -/* A range of pages with a lock */ +/* A FUSE_DLM_WB_LOCK request in flight, on cache->pending */ struct fuse_dlm_range { - /* Interval tree node */ - struct rb_node rb; - /* Start page offset (inclusive) */ + /* The range asked for, as byte offsets, both inclusive */ uint64_t start; - /* End page offset (inclusive) */ uint64_t end; - /* Subtree end value for interval tree */ - uint64_t __subtree_end; - /* Lock mode */ - enum fuse_page_lock_mode mode; - /* Temporary list entry for operations */ + /* A revoke overlapped this request in flight */ + bool killed; + /* The cache->pending link */ struct list_head list; }; -/* Lock modes for FUSE page cache */ -#define FUSE_PCACHE_LK_READ 1 /* Shared read lock */ -#define FUSE_PCACHE_LK_WRITE 2 /* Exclusive write lock */ - -/* Interval tree definitions for page ranges */ -static inline uint64_t fuse_dlm_range_start(struct fuse_dlm_range *range) -{ - return range->start; -} - -static inline uint64_t fuse_dlm_range_last(struct fuse_dlm_range *range) +/* + * Bit of the page at @off within its shard. Callers pass page aligned + * bounds: a grant is aligned in __fuse_get_dlm_lock(), a revoke in + * fuse_dlm_revoke_inval_range() and a query in fuse_dlm_lock_is_held(). + */ +static unsigned long fuse_dlm_bit(uint64_t off) { - return range->end; + return (off & (FUSE_DLM_SHARD_SIZE - 1)) >> PAGE_SHIFT; } -INTERVAL_TREE_DEFINE(struct fuse_dlm_range, rb, uint64_t, __subtree_end, - fuse_dlm_range_start, fuse_dlm_range_last, static, - fuse_page_it); - /** - * fuse_page_cache_init - Initialize a page cache lock manager - * @cache: The cache to initialize + * fuse_dlm_shard_get - the shard covering @off, created if there is none + * @cache: the page cache + * @off: byte offset the caller is about to record a grant at * - * Initialize a page cache lock manager for a FUSE inode. + * Only the recording path needs a shard to exist; readers and the revoke + * paths treat a missing one as a region holding no grant. * - * Return: 0 on success, negative error code on failure + * Return: the shard, or NULL if it could not be allocated. */ -int fuse_dlm_cache_init(struct fuse_inode *inode) +static struct fuse_dlm_shard *fuse_dlm_shard_get(struct fuse_dlm_cache *cache, + uint64_t off) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + unsigned long idx = off >> FUSE_DLM_SHARD_SHIFT; + struct fuse_dlm_shard *shard, *old; - if (!cache) - return -EINVAL; + shard = xa_load(&cache->shards, idx); + if (shard) + return shard; - init_rwsem(&cache->lock); - cache->ranges = RB_ROOT_CACHED; - cache->revoke_gen = 0; + shard = kzalloc(sizeof(*shard), GFP_NOFS); + if (!shard) + return NULL; - return 0; + /* + * Two recorders can reach the same empty region at once; the loser + * drops its shard and takes the winner's. + */ + old = xa_cmpxchg(&cache->shards, idx, NULL, shard, GFP_NOFS); + if (old) { + kfree(shard); + /* xa_cmpxchg() returns an errno as an internal entry */ + return xa_is_err(old) ? NULL : old; + } + + return shard; } /** - * fuse_page_cache_destroy - Clean up a page cache lock manager - * @cache: The cache to clean up + * fuse_dlm_kill_pending - mark in-flight requests overlapping [start, end] + * @cache: The page cache + * @start: Start byte offset of the revoked region + * @end: End byte offset of the revoked region * - * Release all locks and free all resources associated with the cache. + * A revoke overlapping a request still on the wire has nothing to remove + * from the tree, since that grant is not recorded yet. Marking it makes + * fuse_dlm_request_commit() drop the grant instead of recording it. + * + * The nodes are owned by the threads waiting on their replies: mark + * only, never remove or free. + * + * Takes @cache->pending_lock. A request published after this returns is + * one whose FUSE_DLM_WB_LOCK had not been sent when the revoke was + * processed, so the grant it goes on to receive answers a request made + * after the revoke and is recorded, not killed. */ -void fuse_dlm_cache_release_locks(struct fuse_inode *inode) +static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, + uint64_t start, uint64_t end) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - struct rb_node *node; + struct fuse_dlm_range *req; - if (!cache) - return; - - /* Release all locks */ - down_write(&cache->lock); - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - while ((node = rb_first_cached(&cache->ranges)) != NULL) { - range = rb_entry(node, struct fuse_dlm_range, rb); - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } - up_write(&cache->lock); + spin_lock(&cache->pending_lock); + list_for_each_entry(req, &cache->pending, list) + if (req->start <= end && start <= req->end) + req->killed = true; + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_find_overlapping - Find a range that overlaps with [start, end] - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * - * Return: Pointer to the first overlapping range, or NULL if none found + * fuse_dlm_cache_init - Initialize a page cache lock manager + * @inode: The fuse inode to initialize the cache of */ -static struct fuse_dlm_range * -fuse_dlm_find_overlapping(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_cache_init(struct fuse_inode *inode) { - return fuse_page_it_iter_first(&cache->ranges, start, end); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + init_rwsem(&cache->lock); + xa_init(&cache->shards); + spin_lock_init(&cache->pending_lock); + INIT_LIST_HEAD(&cache->pending); } /** - * fuse_page_try_merge - Try to merge ranges within a specific region - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_cache_release_locks - Clean up a page cache lock manager + * @inode: The fuse inode to clean up the cache of * - * Attempt to merge ranges within and adjacent to the specified region - * that have the same lock mode. + * Release all locks and free all resources associated with the cache. */ -static void fuse_dlm_try_merge(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +void fuse_dlm_cache_release_locks(struct fuse_inode *inode) { - struct fuse_dlm_range *range, *next; - uint64_t first = start ? start - 1 : start; - uint64_t last = end < U64_MAX ? end + 1 : end; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache) - return; + /* + * Coverage goes away here, so the whole cache is taken for write: + * see the locking comment on struct fuse_dlm_cache. + */ + down_write(&cache->lock); + /* + * Every grant goes, so every request in flight is revoked. Mark + * only; each node is owned by the thread waiting on its reply. + */ + fuse_dlm_kill_pending(cache, 0, U64_MAX); + xa_for_each(&cache->shards, idx, shard) + kfree(shard); /* - * Find the first range that might need merging. Directly adjacent - * ranges can merge, hence the region is widened by one unit to each - * side (saturating at the type bounds). This must stay an - * interval-tree lookup: the tree holds every cached grant of the - * inode and strided writers grow it for the lifetime of the file, - * so seeding the merge by walking from the tree minimum would make - * every new grant cost a full scan. + * Leaves the xarray empty and usable: an inode is released again + * on every O_TRUNC open, not only on eviction. */ - range = fuse_page_it_iter_first(&cache->ranges, first, last); - - /* Try to merge ranges in and around the specified region */ - while (range && range->start <= last) { - /* Get next range before we potentially modify the tree */ - next = NULL; - if (rb_next(&range->rb)) { - next = rb_entry(rb_next(&range->rb), - struct fuse_dlm_range, rb); - } + xa_destroy(&cache->shards); + up_write(&cache->lock); +} - /* Try to merge with next range if adjacent and same mode */ - if (next && range->mode == next->mode && - range->end + 1 == next->start) { - /* Merge ranges: re-insert so __subtree_end is updated */ - fuse_page_it_remove(next, &cache->ranges); - fuse_page_it_remove(range, &cache->ranges); - range->end = next->end; - fuse_page_it_insert(range, &cache->ranges); - kfree(next); - - /* Continue with the same range */ - continue; - } +/** + * fuse_dlm_shard_record - record a grant over [@start, @end] in @shard + * @shard: the shard covering [@start, @end] + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * @mode: the mode it was granted in + * + * A write grant sets both maps, so a read query is answered by @granted + * alone, and a write grant over a page already held for read upgrades it + * by setting the bit the read grant left clear. + * + * @granted is set before @write, so a query racing this sees the page + * covered for read before it sees it covered for write. Either order is + * safe, a bit not yet seen only costing a re-request of a range already + * held, but this one never reports a write grant the read map does not + * back. + * + * [@start, @end] must be page aligned and lie wholly inside @shard. + * Caller holds fuse_dlm_cache.lock for read. + */ +static void fuse_dlm_shard_record(struct fuse_dlm_shard *shard, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) +{ + unsigned long bit = fuse_dlm_bit(start); + unsigned long last = fuse_dlm_bit(end); - /* Move to next range */ - range = next; + for (; bit <= last; bit++) { + set_bit(bit, shard->granted); + if (mode == FUSE_PAGE_LOCK_WRITE) + set_bit(bit, shard->write); } } /** - * __fuse_dlm_lock_range - Lock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode (read or write) - * @genp: If non-NULL, the revocation generation sampled before the grant - * was requested; recording fails with -EAGAIN if it has moved - * - * Add a locked range on the specified range of pages. - * If parts of the range are already locked, only add the remaining parts. - * For overlapping ranges, handle lock compatibility: - * - READ locks are compatible with existing READ locks - * - READ locks are compatible with existing WRITE locks (downgrade not needed) - * - WRITE locks need to upgrade existing READ locks + * fuse_dlm_record_grant - record a grant across the shards it spans + * @cache: the page cache + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode it was granted in * - * Return: 0 on success, negative error code on failure + * Each shard is filled in on its own: coverage only grows here, and a + * walker under @cache->lock held for read may see part of the grant + * before the rest, which makes it ask again for a range it already has + * rather than trust one it has not got. + * + * Caller holds @cache->lock for read. + * + * Return: 0 on success, negative error code on failure. A failure part + * way through leaves the shards already done recorded, which under + * reports the grant and is safe. */ -static int __fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - const uint64_t *genp) +static int fuse_dlm_record_grant(struct fuse_dlm_cache *cache, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *new_range, *next; - int lock_mode; - bool covered_to_end = false; - int ret = 0; - LIST_HEAD(to_lock); - LIST_HEAD(to_upgrade); - uint64_t current_start = start; + unsigned long idx, last_idx; - if (!cache || start > end) + if (start > end) return -EINVAL; - /* Convert to lock mode */ - lock_mode = (mode == FUSE_PAGE_LOCK_READ) ? FUSE_PCACHE_LK_READ : - FUSE_PCACHE_LK_WRITE; - - down_write(&cache->lock); - - /* - * A revoke was processed after @genp was sampled; the grant this - * record carries may be the very one it targeted (a revoke of a - * not-yet-recorded grant removes nothing and would never be - * retried). Refuse, the caller re-requests. - */ - if (genp && cache->revoke_gen != *genp) { - up_write(&cache->lock); - return -EAGAIN; - } - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check lock compatibility */ - if (lock_mode == FUSE_PCACHE_LK_WRITE && - lock_mode != range->mode) { - /* we own the lock but have to update it. */ - list_add_tail(&range->list, &to_upgrade); - } - /* If WRITE lock already exists - nothing to do */ - - /* If there's a gap before this range, we need to add the missing range */ - if (current_start < range->start) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = range->start - 1; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); - - list_add_tail(&new_range->list, &to_lock); - } - - /* Move current_start past this range */ - if (range->end >= end) - covered_to_end = true; - else - current_start = max(current_start, range->end + 1); + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - /* Move to next range */ - range = next; - } - - /* If there's a gap after the last range to the end, extend the range */ - if (!covered_to_end && current_start <= end) { - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out_free; - } - - new_range->start = current_start; - new_range->end = end; - new_range->mode = lock_mode; - INIT_LIST_HEAD(&new_range->list); + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard; + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - list_add_tail(&new_range->list, &to_lock); - } - - /* update locks, if any lock is in this list it has the wrong mode */ - list_for_each_entry(range, &to_upgrade, list) { - /* Update the lock mode */ - range->mode = lock_mode; - } + shard = fuse_dlm_shard_get(cache, lo); + if (!shard) + return -ENOMEM; - /* Add all new ranges to the tree */ - list_for_each_entry(new_range, &to_lock, list) { - /* Add to interval tree */ - fuse_page_it_insert(new_range, &cache->ranges); + fuse_dlm_shard_record(shard, lo, hi, mode); } - /* Try to merge adjacent ranges with the same mode */ - fuse_dlm_try_merge(cache, start, end); - - up_write(&cache->lock); return 0; - -out_free: - /* Free any ranges we allocated but didn't insert */ - while (!list_empty(&to_lock)) { - new_range = - list_first_entry(&to_lock, struct fuse_dlm_range, list); - list_del(&new_range->list); - kfree(new_range); - } - - /* Restore original lock modes for any partially upgraded locks */ - list_for_each_entry(range, &to_upgrade, list) { - if (lock_mode == FUSE_PCACHE_LK_WRITE) { - /* We upgraded this lock but failed later, downgrade it back */ - range->mode = FUSE_PCACHE_LK_READ; - } - } - - up_write(&cache->lock); - return ret; -} - -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) -{ - return __fuse_dlm_lock_range(inode, start, end, mode, NULL); -} - -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen) -{ - return __fuse_dlm_lock_range(inode, start, end, mode, &gen); } /** - * fuse_dlm_revoke_gen - sample the revocation generation + * fuse_dlm_request_begin - publish a lock request before it is sent * @inode: the fuse inode + * @req: caller-owned storage for the request, live until commit or abort + * @start: start byte offset being requested (inclusive) + * @end: end byte offset being requested (inclusive) + * + * The mode is not recorded here: until the server answers the range is + * held in neither, and the mode that reaches the shards is the one + * passed to fuse_dlm_request_commit(). + * + * A FUSE_DLM_WB_LOCK reply and a NOTIFY revoke are serviced on different + * threads, so a revoke can be processed before the grant the reply + * carries is recorded. Publishing the request before it leaves gives + * that revoke a node to mark; without one it removes nothing, and the + * grant recorded afterwards is never taken back by any later NOTIFY. + * + * The request covers nothing while in flight, so it is kept out of the + * shards. @req is reachable only through cache->pending, which both + * fuse_dlm_request_commit() and fuse_dlm_request_abort() unlink before + * the caller returns; stack storage is therefore fine and nothing is + * allocated here. * - * Sampled before a FUSE_DLM_WB_LOCK request leaves the client. The - * reply and a NOTIFY revoke can be serviced on different threads, so a - * revoke may be processed between the reply arriving and its grant - * being recorded. fuse_dlm_lock_range_gen() re-checks the generation - * under the cache lock and refuses to record a grant such a revoke may - * have already killed. + * Publishing touches the pending list and nothing else, so it takes + * @cache->pending_lock alone. It deliberately does not take + * @cache->lock: this runs on every cached write that is not already + * covered, and taking the cache rwsem for write here made every writer + * of a file queue behind every other one before its request had even + * been sent. */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode) +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end) { - return READ_ONCE(inode->dlm_locked_areas.revoke_gen); + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + req->start = start; + req->end = end; + req->killed = false; + + spin_lock(&cache->pending_lock); + list_add_tail(&req->list, &cache->pending); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_punch_hole - Punch a hole in a locked range - * @cache: The page cache - * @start: Start page offset of the hole - * @end: End page offset of the hole + * fuse_dlm_request_commit - retire a request and record its grant + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * @start: start byte offset the server granted (inclusive) + * @end: end byte offset the server granted (inclusive) + * @mode: the mode that was requested * - * Create a hole in a locked range by splitting it into two ranges. + * Unlinking @req and recording the grant are one step under + * @cache->lock held for read, so a revoke - which takes it for write - + * lands either before it and is seen on @req, or after it and finds the + * grant in the tree. * - * Return: 0 on success, negative error code on failure + * @req is retired in every case and may be reused. + * + * Return: -EAGAIN if a revoke overlapped @req while it was in flight, + * nothing recorded; otherwise the result of recording the grant. */ -static int fuse_dlm_punch_hole(struct fuse_dlm_cache *cache, uint64_t start, - uint64_t end) +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode) { - struct fuse_dlm_range *range, *new_range; + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool revoked; int ret = 0; - if (!cache || start > end) - return -EINVAL; - - /* Find a range that contains [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - if (!range) { - ret = -EINVAL; - goto out; - } - - /* If the hole is at the beginning of the range */ - if (start == range->start) { - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } - - /* If the hole is at the end of the range */ - if (end == range->end) { - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - goto out; - } + /* + * Read, not write: recording adds coverage. Holding it across the + * unlink and the record is what keeps a revoke from landing + * between them, since the revoke paths take it for write. + */ + down_read(&cache->lock); - /* The hole is in the middle, need to split */ - new_range = kmalloc(sizeof(*new_range), GFP_KERNEL); - if (!new_range) { - ret = -ENOMEM; - goto out; - } + spin_lock(&cache->pending_lock); + list_del(&req->list); + revoked = req->killed; + spin_unlock(&cache->pending_lock); - /* Copy properties from original range */ - *new_range = *range; - INIT_LIST_HEAD(&new_range->list); + if (!revoked) + ret = fuse_dlm_record_grant(cache, start, end, mode); + up_read(&cache->lock); - /* Adjust ranges */ - new_range->start = end + 1; - range->end = start - 1; + return revoked ? -EAGAIN : ret; +} - /* Update interval tree */ - fuse_page_it_remove(range, &cache->ranges); - fuse_page_it_insert(range, &cache->ranges); - fuse_page_it_insert(new_range, &cache->ranges); +/** + * fuse_dlm_request_abort - retire a request that got no usable reply + * @inode: the fuse inode + * @req: the request published by fuse_dlm_request_begin() + * + * Nothing is recorded, so a mark left by a revoke does not matter. + */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; -out: - return ret; + spin_lock(&cache->pending_lock); + list_del(&req->list); + spin_unlock(&cache->pending_lock); } /** - * fuse_dlm_unlock_range - Unlock a range of pages - * @cache: The page cache - * @start: Start page offset - * @end: End page offset + * fuse_dlm_unlock_range - Revoke the grants over a range of pages + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * + * The server has taken [start, end] back, so the grants over it are + * removed and the IO paths ask again. Page cache dirtied under a grant + * that has gone is not lost by this: writeback takes the range again for + * every run it sends, and a range it finds unrecorded is a range it asks + * for. * - * Release locks on the specified range of pages. An inverted range is - * rejected rather than silently removing nothing: the callers revoke - * coverage, and a revoke that quietly keeps the grant alive would let - * the re-validating IO paths trust a lock the server has taken away. - * To drop every grant use fuse_dlm_cache_release_locks() (there is no - * in-band sentinel range for it). + * An inverted range is rejected rather than silently revoking nothing: + * the callers revoke coverage, and a revoke that quietly keeps the grant + * alive would let the re-validating IO paths trust a lock the server has + * taken away. To drop every grant use fuse_dlm_cache_release_locks() + * (there is no in-band sentinel range for it). * * Return: 0 on success, negative error code on failure */ -int fuse_dlm_unlock_range(struct fuse_inode *inode, - uint64_t start, uint64_t end) +int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, + uint64_t end) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range, *next; - int ret = 0; + struct fuse_dlm_shard *shard; + unsigned long idx; - if (!cache || start > end) + if (start > end) return -EINVAL; + /* + * Write, not read: this is a path that takes coverage away, and + * the shard walkers rely on that never happening under them. See + * the locking comment on struct fuse_dlm_cache. + */ down_write(&cache->lock); /* - * Unconditional, even when nothing overlaps: the revoke racing - * with an in-flight grant finds an empty tree precisely because - * the grant is not recorded yet, and the bump is what makes the - * recording side notice (see fuse_dlm_lock_range_gen()). + * Before touching any map, and even when nothing is covered: a + * revoke racing an in-flight grant finds nothing set, because that + * grant is not recorded yet. */ - WRITE_ONCE(cache->revoke_gen, cache->revoke_gen + 1); - - /* Find all ranges that overlap with [start, end] */ - range = fuse_page_it_iter_first(&cache->ranges, start, end); - while (range) { - /* Get next overlapping range before we potentially modify the tree */ - next = fuse_page_it_iter_next(range, start, end); - - /* Check if we need to punch a hole */ - if (start > range->start && end < range->end) { - /* Punch a hole in the middle */ - ret = fuse_dlm_punch_hole(cache, start, end); - if (ret) - goto out; - /* After punching a hole, we're done */ - break; - } else if (start > range->start) { - /* Adjust the end of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->end = start - 1; - fuse_page_it_insert(range, &cache->ranges); - } else if (end < range->end) { - /* Adjust the start of the range */ - fuse_page_it_remove(range, &cache->ranges); - range->start = end + 1; - fuse_page_it_insert(range, &cache->ranges); - } else { - /* Complete overlap, remove the range */ - fuse_page_it_remove(range, &cache->ranges); - kfree(range); - } + fuse_dlm_kill_pending(cache, start, end); - range = next; + /* + * Only the regions that hold something, not every index in the + * range: a revoke to EOF runs to U64_MAX and walking that index + * by index would never finish. + */ + xa_for_each_range(&cache->shards, idx, shard, + start >> FUSE_DLM_SHARD_SHIFT, + end >> FUSE_DLM_SHARD_SHIFT) { + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); + unsigned long first = fuse_dlm_bit(lo); + unsigned long nbits = fuse_dlm_bit(hi) - first + 1; + + /* + * Plain, not atomic: @cache->lock is held for write, so no + * reader and no recorder can be looking at these words. + */ + bitmap_clear(shard->granted, first, nbits); + bitmap_clear(shard->write, first, nbits); + + /* + * Keep the shard table to the regions that hold something: + * a file revoked a region at a time would otherwise leave an + * empty shard behind for every one of them. + */ + if (bitmap_empty(shard->granted, FUSE_DLM_SHARD_PAGES)) { + xa_erase(&cache->shards, idx); + kfree(shard); + } } -out: up_write(&cache->lock); - return ret; + return 0; } -/** - * fuse_dlm_range_is_locked - Check if a page range is already locked - * @cache: The page cache - * @start: Start page offset - * @end: End page offset - * @mode: Lock mode to check for (or NULL to check for any lock) +/* + * Is every page of [@from, @to] covered in @mode? Both bounds are page + * aligned and lie inside @shard. A write grant sets both maps, so a + * read request is answered by @granted alone. * - * Check if the specified range of pages is already locked. - * The entire range must be locked for this to return true. + * No lock of its own: bits are set atomically and cleared only under + * fuse_dlm_cache.lock held for write, which the caller holds for read. + * A bit set concurrently may be missed, which costs a re-request of a + * range already held. + */ +static bool fuse_dlm_shard_covers(struct fuse_dlm_shard *shard, uint64_t from, + uint64_t to, enum fuse_page_lock_mode mode) +{ + const unsigned long *map = mode == FUSE_PAGE_LOCK_WRITE ? + shard->write : shard->granted; + unsigned long first = fuse_dlm_bit(from); + unsigned long last = fuse_dlm_bit(to); + + return find_next_zero_bit(map, last + 1, first) > last; +} + +/** + * fuse_dlm_range_is_locked - Check if a byte range is already locked + * @inode: The fuse inode + * @start: Start byte offset + * @end: End byte offset + * @mode: Lock mode to check for * * Return: true if the entire range is locked, false otherwise */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode) +static bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, + uint64_t end, + enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - struct fuse_dlm_range *range; - int lock_mode = 0; - uint64_t current_start = start; + unsigned long idx, last_idx; + bool covered = true; - if (!cache || start > end) + if (start > end) return false; - /* Convert to lock mode if specified */ - if (mode == FUSE_PAGE_LOCK_READ) - lock_mode = FUSE_PCACHE_LK_READ; - else if (mode == FUSE_PAGE_LOCK_WRITE) - lock_mode = FUSE_PCACHE_LK_WRITE; - + /* + * Read: coverage is only ever removed under @cache->lock held for + * write, so the set of grants can grow under this walk but never + * shrink. That is what lets the shards be visited one at a time, + * and their maps read without any further lock. The worst a + * concurrent recorder can do is make this report a range uncovered + * that has just become covered, and the caller then asks for a + * grant it already holds. + */ down_read(&cache->lock); - /* Find the first range that overlaps with [start, end] */ - range = fuse_dlm_find_overlapping(cache, start, end); - - /* Check if the entire range is covered */ - while (range && current_start <= end) { - /* - * The held lock must be at least as strong as the one - * requested. A WRITE lock (exclusive) satisfies a READ - * request, so only treat the range as uncovered when the - * held mode is weaker than what we ask for. This avoids - * re-requesting a READ lock for a range we already hold - * a WRITE lock on (e.g. read-after-write). - */ - if (lock_mode && range->mode < lock_mode) { - /* Held lock is weaker than requested */ - up_read(&cache->lock); - return false; - } - - /* Check if there's a gap before this range */ - if (current_start < range->start) { - /* Found a gap */ - up_read(&cache->lock); - return false; - } - - /* Covered through the end of the requested range? */ - if (range->end >= end) { - up_read(&cache->lock); - return true; - } - - /* Move current_start past this range */ - current_start = range->end + 1; - - /* Get next overlapping range */ - range = fuse_page_it_iter_next(range, start, end); - } - - /* Check if we covered the entire range */ - if (current_start <= end) { - /* There's a gap at the end */ - up_read(&cache->lock); - return false; - } + last_idx = end >> FUSE_DLM_SHARD_SHIFT; - up_read(&cache->lock); - return true; -} - -/** - * fuse_dlm_write_grant_exists - does the inode hold an exclusive grant anywhere - * @fi: the fuse inode - * - * Unlike fuse_dlm_range_is_locked(), which asks whether one range is fully - * covered, this asks whether any part of the file is held exclusively. A - * client that holds a write grant may be sitting on dirty page cache the - * server has not seen, so its mtime and ctime run ahead of anything the - * server can report. - * - * Return: true if at least one recorded range is held for write - */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *fi) -{ - struct fuse_dlm_cache *cache = &fi->dlm_locked_areas; - struct fuse_dlm_range *range; - bool held = false; + for (idx = start >> FUSE_DLM_SHARD_SHIFT; idx <= last_idx; idx++) { + struct fuse_dlm_shard *shard = xa_load(&cache->shards, idx); + uint64_t lo = max(start, FUSE_DLM_SHARD_FIRST(idx)); + uint64_t hi = min(end, FUSE_DLM_SHARD_LAST(idx)); - down_read(&cache->lock); - for (range = fuse_dlm_find_overlapping(cache, 0, U64_MAX); range; - range = fuse_page_it_iter_next(range, 0, U64_MAX)) { - if (range->mode == FUSE_PCACHE_LK_WRITE) { - held = true; + if (!shard || !fuse_dlm_shard_covers(shard, lo, hi, mode)) { + covered = false; break; } } + up_read(&cache->lock); - return held; + return covered; } /** @@ -634,11 +559,10 @@ bool fuse_dlm_lock_is_held(struct fuse_inode *fi, loff_t offset, * re-validating the grant must not re-request on a nonzero return or * they would spin. */ -int fuse_get_dlm_lock(struct file *file, loff_t offset, - size_t length, enum fuse_page_lock_mode mode) +static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, + loff_t offset, size_t length, + enum fuse_page_lock_mode mode) { - struct fuse_file *ff = file->private_data; - struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_mount *fm = ff->fm; @@ -646,13 +570,24 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, FUSE_ARGS(args); struct fuse_dlm_lock_in inarg; struct fuse_dlm_lock_out outarg; - uint64_t gen; + struct fuse_dlm_range req; + uint64_t pg_start, pg_end; + uint64_t grant_start, grant_end; + int tries = FUSE_DLM_GRANT_RETRIES; int err; /* An empty range needs no lock. */ if (!length) return 0; + /* + * note that the offset and length don't have to be page aligned + * here but since we only get here on writeback caching we will + * send out page aligned requests + */ + pg_start = (uint64_t)offset & PAGE_MASK; + pg_end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + restart: /* note that this can be run from different processes * at the same time. It is intentionally not protected @@ -661,27 +596,20 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, * The early exit uses the same helper the callers re-validate * with, so this check and a later fuse_dlm_lock_is_held() can * never disagree about what counts as covered. */ - if (fuse_dlm_lock_is_held(fi, offset, length, mode)) - return 0; /* we already have this area locked */ - - /* - * Sample the revocation generation before the request leaves. - * The reply and a NOTIFY revoke are serviced on different - * threads, so a revoke aimed at the grant this request returns - * can be processed before the grant is recorded below -- - * recording it anyway would resurrect a dead grant that no later - * NOTIFY will ever remove. - */ - gen = fuse_dlm_revoke_gen(fi); + if (fuse_dlm_lock_is_held(fi, offset, length, mode)) { + /* + * Already covered, and the record says nothing beyond that, + * so this is one shared acquisition end to end. + */ + return 0; + } memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); inarg.fh = ff->fh; - /* note that the offset and length don't have to be page aligned - * here but since we only get here on writeback caching we will - * send out page aligned requests */ - inarg.start = offset & PAGE_MASK; - inarg.end = (offset + length - 1) | (PAGE_SIZE - 1); + inarg.start = pg_start; + inarg.end = pg_end; inarg.type = (mode == FUSE_PAGE_LOCK_WRITE) ? FUSE_DLM_LOCK_WRITE : FUSE_DLM_LOCK_READ; @@ -693,18 +621,23 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, args.out_numargs = 1; args.out_args[0].size = sizeof(outarg); args.out_args[0].value = &outarg; + + /* Publish before sending; see fuse_dlm_request_begin() */ + fuse_dlm_request_begin(fi, &req, inarg.start, inarg.end); + err = fuse_simple_request(fm, &args); - if (err == -ENOSYS) { - /* fuse server does not support dlm, save the info */ - fc->dlm = 0; + if (err) { + fuse_dlm_request_abort(fi, &req); + if (err == -ENOSYS) { + /* fuse server does not support dlm, save the info */ + fc->dlm = 0; + } return err; } - if (err) - return err; - if (inarg.start < outarg.start || inarg.end > outarg.end) { /* fuse server is seriously broken */ + fuse_dlm_request_abort(fi, &req); pr_warn("fuse: dlm lock request for %llu:%llu returned %llu:%llu bytes\n", inarg.start, inarg.end, outarg.start, outarg.end); fuse_abort_conn(fc); @@ -712,21 +645,54 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, } /* - * The server granted the lock; record it so - * fuse_dlm_lock_is_held() sees it. + * Keep the recorded grant to a bounded distance either side of + * what was asked for; see FUSE_DLM_MAX_EXTRA_GRANT. Both bounds + * stay outside [pg_start, pg_end], so the range this call has to + * cover is still covered. + */ + grant_start = outarg.start; + grant_end = outarg.end; + /* + * Both differences are safe: the check above established + * outarg.start <= pg_start <= pg_end <= outarg.end, and a branch + * is only taken when there is more than the cap to give back, so + * neither adjusted bound can wrap. + */ + if (pg_start - grant_start > FUSE_DLM_MAX_EXTRA_GRANT) + grant_start = pg_start - FUSE_DLM_MAX_EXTRA_GRANT; + if (grant_end - pg_end > FUSE_DLM_MAX_EXTRA_GRANT) + grant_end = pg_end + FUSE_DLM_MAX_EXTRA_GRANT; + + /* + * Align inward. A page is covered only when it is covered whole, + * and the bit helpers take that as given; the bounds themselves are + * the server's to choose, only their superset property having been + * checked. Rounding inward cannot uncover [pg_start, pg_end], + * which is page aligned already. */ - err = fuse_dlm_lock_range_gen(fi, outarg.start, outarg.end, mode, gen); + grant_start = ALIGN(grant_start, PAGE_SIZE); + /* A last-byte offset, so it is the successor that aligns */ + grant_end -= (grant_end + 1) & (PAGE_SIZE - 1); + + /* Retire the request and record the grant */ + err = fuse_dlm_request_commit(fi, &req, grant_start, grant_end, mode); if (err == -EAGAIN) { /* - * A revoke was processed while the request was in flight; - * the grant may already be dead, so re-request instead of - * recording it. Retry until a grant survives long enough to - * be recorded: giving up here would hand the caller an error - * for a range no one else holds, and the write path turns - * that into a failed write. Each pass makes a fresh server - * round trip, so a revoke storm throttles this loop rather - * than spinning it. + * A revoke overlapping this range was processed while the + * request was in flight, so the grant is dead. Retry + * rather than fail: no one else holds the range, and the + * write path turns an error into a failed write. + * + * Not forever, though. Every pass is a whole round trip, + * which throttles the loop but does not end it, and + * writeback asks for a grant with a folio locked, so a node + * revoking as fast as the grants arrive would hold that + * folio and this task for as long as it kept going. */ + if (fatal_signal_pending(current)) + return -EINTR; + if (!tries--) + return -EIO; goto restart; } @@ -743,3 +709,32 @@ int fuse_get_dlm_lock(struct file *file, loff_t offset, return 0; } + +int fuse_get_dlm_lock(struct file *file, loff_t offset, + size_t length, enum fuse_page_lock_mode mode) +{ + return __fuse_get_dlm_lock(file->private_data, file_inode(file), + offset, length, mode); +} + +/** + * fuse_dlm_regrant_range - hold [start, end] again for writeback + * @ff: a fuse file open for writing on @inode + * @inode: the inode + * @start: start byte offset (inclusive) + * @end: end byte offset (inclusive) + * + * Writeback holds the range again before sending a folio, since a revoke + * may have arrived between the write and the send. Whatever the other + * holder wrote in between is overwritten, which for two writers that + * never synchronised is a legitimate order. + * + * A range still held is the ordinary case: the grant is found recorded + * and nothing is sent to the server. + */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end) +{ + return __fuse_get_dlm_lock(ff, inode, start, end - start + 1, + FUSE_PAGE_LOCK_WRITE); +} diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index 30fdbb26bd3daf..c54cad9d000226 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -7,12 +7,17 @@ #define _FS_FUSE_DLM_CACHE_H #include -#include +#include #include +#include #include +#include +#include struct fuse_inode; +struct fuse_dlm_range; +struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; @@ -26,53 +31,133 @@ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; */ #define FUSE_DLM_GRANT_UNRECORDED 1 -/* Page cache lock manager */ +/* + * Coverage is kept per aligned region of the file rather than in one + * structure for the whole inode, because one structure needs one lock + * and every thread writing the file then serialises on it however far + * apart their ranges are. The region wants to be small enough that + * concurrent writers land in different ones and large enough that the + * per-region overhead stays amortised. + */ +#define FUSE_DLM_SHARD_SHIFT 24 +#define FUSE_DLM_SHARD_SIZE (1ULL << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_PAGES (FUSE_DLM_SHARD_SIZE / PAGE_SIZE) + +/* First and last byte offset (both inclusive) covered by shard @idx */ +#define FUSE_DLM_SHARD_FIRST(idx) ((uint64_t)(idx) << FUSE_DLM_SHARD_SHIFT) +#define FUSE_DLM_SHARD_LAST(idx) (FUSE_DLM_SHARD_FIRST(idx) + \ + FUSE_DLM_SHARD_SIZE - 1) + +/* + * The grants over one region, a bit per page. + * + * @granted says the page is covered, @write that it is covered for + * write. A write grant sets both, so a read request is answered by + * @granted alone and nothing has to compare modes; @write is a subset of + * @granted. + * + * A region is a fixed span of pages, so the maps are a fixed size and + * recording a grant neither allocates nor rearranges anything: adjacent + * grants coalesce because they set neighbouring bits, and a region + * fragmented to the last page costs no more than the two maps it already + * has. + * + * Bits are set with the atomic helpers, since recorders run concurrently + * under fuse_dlm_cache.lock held for read. They are cleared only under + * that lock held for write, which excludes every reader and every + * recorder, so the revoke path uses the plain bulk helpers and a query + * needs no lock of its own. + */ +struct fuse_dlm_shard { + unsigned long granted[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; + unsigned long write[BITS_TO_LONGS(FUSE_DLM_SHARD_PAGES)]; +}; + +/* + * Page cache lock manager. + * + * The shards hold the grants the client has been given. A request still + * on the wire covers nothing and lives on @pending instead, so the + * shards answer for grants only. See struct fuse_dlm_range in + * fuse_dlm_cache.c. + * + * Locking, outermost first: + * + * @lock rw_semaphore over the whole cache. Taken for read by + * everything that adds or reads coverage, and for write + * only by the paths that take coverage away + * (fuse_dlm_unlock_range, fuse_dlm_cache_release_locks). + * That is the invariant the shard walks rely on: + * **coverage is only ever removed under @lock held for + * write**, so a walker holding it for read sees a set of + * grants that can grow under it but never shrink, and may + * therefore visit shards one at a time and read their + * maps without any further lock. Shards are freed only + * there too, so a shard pointer stays good for as long as + * the read side is held. + * @pending_lock the pending list. Innermost, and the only lock + * fuse_dlm_request_begin() and fuse_dlm_request_abort() + * take at all. + */ struct fuse_dlm_cache { - /* Lock protecting the tree */ + /* See the locking comment above */ struct rw_semaphore lock; - /* Interval tree of locked ranges */ - struct rb_root_cached ranges; /* - * Bumped under @lock by every revocation - * (fuse_dlm_unlock_range(), fuse_dlm_cache_release_locks()); - * lets fuse_get_dlm_lock() order recording a reply's grant - * against revokes processed while the reply was in flight. + * struct fuse_dlm_shard by offset >> FUSE_DLM_SHARD_SHIFT, + * allocated when a region first holds a grant. + */ + struct xarray shards; + /* Protects @pending and the killed flag of everything on it */ + spinlock_t pending_lock; + /* + * FUSE_DLM_WB_LOCK requests in flight. Owned by the queueing + * thread; the revoke paths only mark them killed. */ - uint64_t revoke_gen; + struct list_head pending; }; /* Initialize a page cache lock manager */ -int fuse_dlm_cache_init(struct fuse_inode *inode); +void fuse_dlm_cache_init(struct fuse_inode *inode); /* Clean up a page cache lock manager */ void fuse_dlm_cache_release_locks(struct fuse_inode *inode); -/* Lock a range of pages */ -int fuse_dlm_lock_range(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); +/* + * Publish a FUSE_DLM_WB_LOCK for [start, end] before it is sent, so a + * revoke processed while the reply is on the wire can mark it. @req is + * caller-owned storage, live until the matching commit or abort. The + * mode is not recorded until the grant is, so only the commit takes it. + */ +void fuse_dlm_request_begin(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end); -/* As above, but refuse (-EAGAIN) if a revoke ran since @gen was sampled */ -int fuse_dlm_lock_range_gen(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode, - uint64_t gen); +/* + * Retire @req and record the grant [start, end] as one step under the + * cache lock. -EAGAIN means a revoke overlapped @req in flight and + * nothing was recorded; the caller must request again. @req is retired + * either way. + */ +int fuse_dlm_request_commit(struct fuse_inode *inode, + struct fuse_dlm_range *req, uint64_t start, + uint64_t end, enum fuse_page_lock_mode mode); -/* Sample the revocation generation (see fuse_dlm_lock_range_gen()) */ -uint64_t fuse_dlm_revoke_gen(struct fuse_inode *inode); +/* Retire @req without recording anything */ +void fuse_dlm_request_abort(struct fuse_inode *inode, + struct fuse_dlm_range *req); /* Unlock a range of pages */ int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); -/* Check if a page range is already locked */ -bool fuse_dlm_range_is_locked(struct fuse_inode *inode, uint64_t start, - uint64_t end, enum fuse_page_lock_mode mode); - /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); -/* Is any part of the file held for write? */ -bool fuse_dlm_write_grant_exists(struct fuse_inode *inode); +/* Hold [start, end] again so writeback can send what it found revoked */ +int fuse_dlm_regrant_range(struct fuse_file *ff, struct inode *inode, + uint64_t start, uint64_t end); + /* This is the interface to the filesystem */ int fuse_get_dlm_lock(struct file *file, loff_t offset, diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 31580b7834e296..e3de135c291220 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -202,21 +202,6 @@ struct fuse_inode { /* dlm locked areas we have sent lock requests for */ struct fuse_dlm_cache dlm_locked_areas; - /* - * Serializes buffered-write page-cache dirtying against - * the forced-direct-IO latch transition driven by - * NOTIFY_INVAL_INODE (fuse_reverse_inval_inode()), which - * may be delivered by the same server thread that still - * owes a reply to an in-flight write holding the inode - * lock. The buffered writer holds this for read around - * the dirtying and re-checks the latch under it; the - * NOTIFY latch site takes it for write (trylock, never - * blocking) around its page-cache invalidate + latch set. - * Only regular files initialise it -- it shares storage - * with the readdir-cache union arm. - */ - struct percpu_rw_semaphore *wb_inval_rwsem; - /* * Rate of FUSE_NOTIFY_INVAL_INODE data invalidations * for this whole file: notify_stamp is the jiffies of diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index bb52782c064d3b..7e573b4e7e4004 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -220,23 +220,6 @@ static void fuse_evict_inode(struct inode *inode) WARN_ON(!list_empty(&fi->queued_writes)); fuse_dlm_cache_release_locks(fi); } - - /* - * Free the coherency gate here rather than in ->free_inode: that runs - * from an RCU callback, where percpu_free_rwsem() may sleep in - * rcu_sync_dtor() if the write side has not fully quiesced. No user - * can remain by eviction time: gate readers hold a file reference and - * a concurrent notify holds an inode reference. wb_inval_rwsem lives - * in the regular-file union arm and is only ever allocated for regular - * files, so gate on S_ISREG (but not fuse_is_bad() -- bad-marked - * regular files still own a gate); a directory's overlapping - * readdir-cache fields must not be misread. - */ - if (S_ISREG(inode->i_mode) && fi->wb_inval_rwsem) { - percpu_free_rwsem(fi->wb_inval_rwsem); - kfree(fi->wb_inval_rwsem); - fi->wb_inval_rwsem = NULL; - } } static int fuse_reconfigure(struct fs_context *fsc) @@ -536,14 +519,14 @@ u32 fuse_get_cache_mask(struct inode *inode) * for exactly what the grant covers: * * - size, when the server reports less than i_size and the tail it does not - * know about, [srv_size, i_size), is entirely under a write grant. Taking - * the server's answer would shrink i_size and have truncate_pagecache() - * throw the unwritten tail away. - * - mtime and ctime, while a write grant covers unwritten data: our writes - * have stamped them locally and the server's stamps predate them. Only - * while the cache is actually dirty, not for as long as the grant lives: - * a grant is held until it is revoked or the inode is evicted, and past - * the writeback the server's stamps are the newer ones. Keeping ours + * know about, [attr->size, i_size), is entirely under a write grant. + * Taking the server's answer would shrink i_size and have + * truncate_pagecache() throw the unwritten tail away. + * - mtime and ctime, while the page cache is dirty or under writeback: our + * writes have stamped them locally and the server's stamps predate them. + * Only while the cache is actually dirty, not for as long as a grant + * lives: a grant is held until it is revoked or the inode is evicted, and + * past the writeback the server's stamps are the newer ones. Keeping ours * beyond that would hide a remote chown or chmod indefinitely. * * A remote truncate cannot slip through. It has to revoke the grant first, @@ -567,16 +550,31 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, !S_ISREG(inode->i_mode)) return cache_mask; - if (!fuse_dlm_write_grant_exists(fi)) - return cache_mask; - + /* + * A dirty mapping keeps the local attributes authoritative even + * when no grant is recorded: a fault dirties pages under a + * page-mkwrite lock that is never recorded, and a truncate revokes + * the tail grants itself while cached writes above the new size + * are still waiting for writeback. + */ if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) cache_mask |= STATX_MTIME | STATX_CTIME; + /* + * The local size stays authoritative while the extension is + * covered by a write grant, and also while anything in + * [attr->size, size) is dirty or under writeback: those bytes + * exist only here, and taking the server's smaller size would + * truncate them away before they are ever sent. The grant check + * alone misses them, because a page-mkwrite grant is never + * recorded and a local truncate revokes its own tail grants. + */ if (have_size && size > (loff_t) attr->size && - fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, - FUSE_PAGE_LOCK_WRITE)) + (fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + FUSE_PAGE_LOCK_WRITE) || + filemap_range_needs_writeback(inode->i_mapping, attr->size, + size - 1))) cache_mask |= STATX_SIZE; return cache_mask; @@ -910,22 +908,28 @@ static void fuse_dlm_revoke_inval_range(struct fuse_inode *fi, loff_t offset, * Drop a page-cache range on behalf of a NOTIFY invalidate. * * invalidate_inode_pages2_range() waits out folios under writeback and - * launders dirty ones, both of which need a FUSE_WRITE reply. While - * writepages are frozen (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, - * pre-SETATTR flush) no reply can arrive, because fuse_flush_writepages() - * parks the request on fi->queued_writes until fuse_release_nowrite(). A - * server that revokes from inside the handler it is revoking for then - * deadlocks against its own reply. fuse_do_setattr() states the same rule - * for its own invalidate. + * launders dirty ones, both of which need a FUSE_WRITE reply. It is only + * needed when the range can hold data the server has not seen. * - * So while frozen use invalidate_mapping_pages(), which skips dirty and - * under-writeback folios and never blocks. The stale clean folios still - * go, and the freezes that span a request drop the cache themselves once - * they complete: fuse_do_setattr() invalidates the mapping after releasing - * the freeze, the O_TRUNC open path calls truncate_pagecache(). + * @may_be_dirty false says it cannot, on the strength of the DLM range + * record: every way a folio gets dirtied under a grant raises that record + * before the data lands, so a range it reports clean has no dirty folio to + * launder. invalidate_mapping_pages() then drops the same folios without + * ever waiting for the server. + * + * The same substitution is forced while writepages are frozen + * (fuse_set_nowrite(): truncate, O_TRUNC open, fsync, pre-SETATTR flush), + * where no reply can arrive because fuse_flush_writepages() parks the + * request on fi->queued_writes until fuse_release_nowrite(). A server that + * revokes from inside the handler it is revoking for would otherwise + * deadlock against its own reply. fuse_do_setattr() states the same rule + * for its own invalidate. There the dirty folios are left behind, and the + * freezes that span a request drop the cache themselves once they complete: + * fuse_do_setattr() invalidates the mapping after releasing the freeze, the + * O_TRUNC open path calls truncate_pagecache(). */ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, - pgoff_t end) + pgoff_t end, bool may_be_dirty) { struct fuse_inode *fi = get_fuse_inode(inode); bool frozen; @@ -934,7 +938,7 @@ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, frozen = fi->writectr < 0; spin_unlock(&fi->lock); - if (frozen) + if (frozen || !may_be_dirty) invalidate_mapping_pages(inode->i_mapping, start, end); else invalidate_inode_pages2_range(inode->i_mapping, start, end); @@ -943,11 +947,12 @@ static void fuse_notify_invalidate_range(struct inode *inode, pgoff_t start, int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, loff_t offset, loff_t len) { - struct percpu_rw_semaphore *wb_sem = NULL; struct fuse_inode *fi; struct inode *inode; + loff_t end_byte; pgoff_t pg_start; pgoff_t pg_end; + bool tracked; inode = fuse_ilookup(fc, nodeid, NULL); if (!inode) @@ -980,57 +985,54 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, else pg_end = (offset + len - 1) >> PAGE_SHIFT; + /* Byte bounds of the same region */ + end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + /* - * A data invalidation means another (remote) entity is modifying - * the file. Two things happen here: + * A data invalidation means another (remote) entity is + * modifying the file. Two things happen here: * - * 1. Coherency. Drop the affected page-cache range so no local - * read returns a folio the remote modify has superseded. This - * runs under the write side of the per-inode coherency gate - * (wb_inval_rwsem), which fences cache-serving buffered reads - * and buffered writes out for the whole invalidate. Unlike the - * old best-effort trylock this BLOCKS -- the notify has - * priority: percpu_down_write() parks new gate readers, drains - * in-flight ones, then invalidates. A blocking writer here is - * safe only under a server that services request replies on - * threads other than the one delivering this notify: the write - * side waits for gate readers to drain, and a cache-miss read - * holds the read side across its FUSE_READ round-trip. redfs' - * dlm server provides that contract; a server that cannot must - * not enable writeback+dlm. + * 1. Coherency. Drop the affected page-cache range so no + * local read returns a folio the remote modify has + * superseded. Nothing is fenced out for it. A read + * racing the drop either misses and refetches or returns + * data that was current when it was copied. A write + * racing it is caught on the way out instead: its bytes + * were recorded before they were dirtied, this revoke + * marks the range rather than forgetting it, and + * writeback holds the range again before sending + * anything it finds marked that way. * - * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), under - * fi->lock, updated for every data invalidation) of how fast - * these arrive; when they come in a rapid stream -- a remote - * writer repeatedly invalidating -- and the inode is also open - * for writing here, latch it into direct IO until the last - * writer closes or it is mmapped. When latched, drop the whole - * mapping rather than just the notified range, or dirty folios - * outside it would be invisible to the forced direct reads - * (stale read / lost write). Latching is opt-in via the - * enable_notify_dio module parameter and off by default; the - * average is kept up to date either way, so enabling it at - * runtime takes effect on the next storm rather than after a - * warm-up. Clearing it at runtime stops new latches but lets + * 2. Latch. Keep a moving average (fuse_notify_inval_hot(), + * under fi->lock, updated for every data invalidation) of + * how fast these arrive; when they come in a rapid stream + * -- a remote writer repeatedly invalidating -- and the + * inode is also open for writing here, latch it into + * direct IO until the last writer closes or it is mmapped. + * When latched, drop the whole mapping rather than just + * the notified range, or dirty folios outside it would be + * invisible to the forced direct reads (stale read / lost + * write). Latching is opt-in via the enable_notify_dio + * module parameter and off by default; the average is kept + * up to date either way, so enabling it at runtime takes + * effect on the next storm rather than after a warm-up. + * Clearing it at runtime stops new latches but lets * already-latched inodes run out on the usual exits (last * writer closes, or mmap). * - * The gate (and the average) exist only for writeback+dlm regular - * files; elsewhere wb_sem is NULL and the invalidate runs - * unserialized (best-effort), as before. An mmapped inode - * keeps the gate -- fuse_cache_read_iter() and - * fuse_cache_write_iter() enter it unconditionally and rely - * on the revoke staying fenced -- but is never latched: - * a mapping needs the page cache, and fuse_file_mmap() - * reverts any latch it races with. + * The average and the latch exist only for writeback+dlm + * regular files; elsewhere there is no record to consult and + * the range is dropped as it always was. An mmapped inode is + * never latched: a mapping needs the page cache, and + * fuse_file_mmap() reverts any latch it races with. */ - if (S_ISREG(inode->i_mode) && fc->writeback_cache && - fc->dlm && !FUSE_IS_DAX(inode) && - !fuse_inode_backing(fi)) - wb_sem = fi->wb_inval_rwsem; + tracked = S_ISREG(inode->i_mode) && fc->writeback_cache && + fc->dlm && !FUSE_IS_DAX(inode) && + !fuse_inode_backing(fi); - if (wb_sem) { + if (tracked) { bool hot, has_writer, latched = false; + bool may_be_dirty, has_pages; spin_lock(&fi->lock); hot = fuse_notify_inval_hot(fi); @@ -1038,22 +1040,42 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, spin_unlock(&fi->lock); /* - * Priority write side: park new gate readers, - * drain in-flight ones, then invalidate. Blocks - * (unlike the old trylock) -- see the contract in - * the comment above. + * What this notify has to do. Nothing cached in the + * range means the drop is a no-op and the revoke is + * the whole job. Otherwise the page cache says + * whether the drop has to launder, which is what + * makes it wait for a FUSE_WRITE reply. */ - percpu_down_write(wb_sem); + has_pages = filemap_range_has_page(inode->i_mapping, + offset, end_byte); + may_be_dirty = filemap_range_needs_writeback( + inode->i_mapping, offset, end_byte); /* - * Revoke the DLM lock range under the gate write - * side, atomically with the page drop: gate readers - * re-validate their grant right after entering, and - * a grant that passed that check must stay visible - * for their whole gate hold. + * Put unwritten data on the server while the grant + * still covers it, rather than leaving it to the drop + * below. After the revoke writeback would have to + * take the range again to send those bytes: a DLM + * round trip from inside the handler the server is + * waiting on. do_writepages() runs in this context, + * so the grant is asked for before the revoke. + * + * Waited out here rather than left to the drop, which + * launders when the record says the range may be dirty + * and so waits for these same replies. One explicit + * wait, before the revoke, and the drop then finds + * nothing under writeback to block on. Either way a + * server that revokes from a thread it also needs to + * answer FUSE_WRITE on deadlocks here, the same + * contract fuse_notify_invalidate_range() states for a + * frozen inode. The error is left to the mapping, + * where fsync collects it. */ - if (fc->dlm && fc->writeback_cache) - fuse_dlm_revoke_inval_range(fi, offset, len); + if (has_pages && may_be_dirty) + filemap_write_and_wait_range(inode->i_mapping, + offset, end_byte); + + fuse_dlm_revoke_inval_range(fi, offset, len); if (enable_notify_dio && hot && has_writer && !mapping_mapped(inode->i_mapping) && @@ -1069,27 +1091,31 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* * Latched: drop the whole mapping (dirty folios * outside the notified range would be invisible to - * the forced direct reads). Otherwise just the - * notified range. + * the forced direct reads), and the record says + * nothing about the rest of the file, so launder. + * Otherwise just the notified range, and only if + * anything is cached there. */ if (fuse_inode_force_dio(inode)) - fuse_notify_invalidate_range(inode, 0, -1); - else + fuse_notify_invalidate_range(inode, 0, -1, true); + else if (has_pages) fuse_notify_invalidate_range(inode, pg_start, - pg_end); - - percpu_up_write(wb_sem); + pg_end, + may_be_dirty); if (latched) pr_info_ratelimited("FUSE: inode %llu latched to direct IO on invalidation notify storm\n", nodeid); } else { - /* No gate on this inode (DAX, backing, non-regular, - * or the gate allocation failed): drop the lock - * range unserialized (best-effort), as before. */ + /* + * No record on this inode (DAX, backing, non-regular, + * or no DLM), so assume the range can hold unwritten + * data and drop it as before. + */ if (fc->dlm && fc->writeback_cache) fuse_dlm_revoke_inval_range(fi, offset, len); - fuse_notify_invalidate_range(inode, pg_start, pg_end); + fuse_notify_invalidate_range(inode, pg_start, pg_end, + true); } } iput(inode); From 7630edf37bb5613ba43cd85a21598bca82bd063b Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 08:15:59 +0200 Subject: [PATCH 04/17] fuse: carry the write retry flag in iomap's private pointer fuse_writeback_write_iter() published its AOP_TRUNCATED_PAGE retry state in a connection-wide xarray keyed by task pointer, and erased it again, once per buffered write. A task pointer is a high index, so every write built and tore down the whole node chain down to it under one lock shared by every writer of the mount, and its GFP_KERNEL allocation ran with i_rwsem held. iomap already carries a private pointer through to fuse_iomap_read_folio_range(), where fuse put the struct file. Put the file and the flag in one struct on the stack and pass that instead. The store could fail and take the write down with it, and the load could come back empty and drop a needed retry. Neither is reachable now. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 63 ++++++++++++++++++------------------------------ fs/fuse/fuse_i.h | 18 -------------- fs/fuse/inode.c | 2 -- 3 files changed, 24 insertions(+), 59 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e21a08dd5704df..4aa884e411e349 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1176,13 +1176,24 @@ static int fuse_read_folio(struct file *file, struct folio *folio) return err; } +/* + * What iomap_file_buffered_write() carries for fuse, reached from the + * read-back callback as iter->private. @file is what that callback needs + * anyway, so the retry flag rides along and lives exactly as long as the + * call, with nothing to allocate or free. + */ +struct fuse_iomap_write_ctx { + struct file *file; + /* fuse_iomap_read_folio_range() hit AOP_TRUNCATED_PAGE */ + bool retry_needed; +}; + static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, struct folio *folio, loff_t pos, size_t len) { - struct file *file = iter->private; - struct inode *inode = file_inode(file); - struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_iomap_write_ctx *ctx = iter->private; + struct file *file = ctx->file; size_t off = offset_in_folio(folio, pos); int ret; @@ -1197,7 +1208,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * * However, iomap doesn't understand AOP_TRUNCATED_PAGE. * We need to: - * 1. Mark the retry flag (caller stored it in xarray) + * 1. Mark the retry flag on the caller's write context * 2. Convert to -EAGAIN so iomap sees an error * 3. Let fuse_cache_write_iter() detect and retry * @@ -1208,13 +1219,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE support. */ if (ret == AOP_TRUNCATED_PAGE) { - struct fuse_dlm_retry *retry; - unsigned long task_key = (unsigned long)current; - - retry = xa_load(&fc->dlm_retry_tasks, task_key); - if (retry) { - retry->retry_needed = true; - } + ctx->retry_needed = true; /* Convert to -EAGAIN for iomap */ ret = -EAGAIN; @@ -1844,30 +1849,15 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, struct iov_iter *from, struct file *file) { - struct fuse_conn *fc = get_fuse_conn(file_inode(file)); - ssize_t written, total_written = 0; - /* * TEMPORARY WORKAROUND for iomap write deadlock: * - * Stack-allocate retry state and register it before calling - * iomap. If fuse_iomap_read_folio_range() encounters - * AOP_TRUNCATED_PAGE, it will mark retry_needed. - * - * Stack allocation ensures no memory leaks - the state is - * valid for the duration of this function call and is - * automatically cleaned up. + * The context fuse_iomap_read_folio_range() marks when it hits + * AOP_TRUNCATED_PAGE. iomap hands it back through its private + * pointer, which fuse needs for @file either way. */ - struct fuse_dlm_retry retry_state = { - .retry_needed = false, - }; - unsigned long task_key = (unsigned long)current; - int xa_ret; - - xa_ret = xa_err(xa_store(&fc->dlm_retry_tasks, task_key, - &retry_state, GFP_KERNEL)); - if (xa_ret) - return xa_ret; + struct fuse_iomap_write_ctx ctx = { .file = file }; + ssize_t written, total_written = 0; retry: /* @@ -1878,14 +1868,14 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * the next iteration with iov_iter already drained, and iomap * would re-enter with len==0 and livelock on a 0-length mapping. */ - retry_state.retry_needed = false; + ctx.retry_needed = false; /* * Use iomap so that we can do granular uptodate reads * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, file); + &fuse_iomap_write_ops, &ctx); if (written > 0) total_written += written; @@ -1897,17 +1887,12 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * The folio has been unlocked by fuse_do_readfolio(), * breaking the ABBA deadlock with page invalidation. * - * Keep the entry in xarray and reuse it for the retry. - * * Remove this when mainline iomap gains AOP_TRUNCATED_PAGE * retry support. */ - if (retry_state.retry_needed && iov_iter_count(from)) + if (ctx.retry_needed && iov_iter_count(from)) goto retry; - /* Remove from xarray now that we're done */ - xa_erase(&fc->dlm_retry_tasks, task_key); - return written < 0 ? written : total_written; } diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index e3de135c291220..6dc2595b2f7450 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -686,17 +686,6 @@ struct fuse_sync_bucket { struct rcu_head rcu; }; -/** - * DLM retry tracking for iomap write deadlock workaround. - * - * Temporary workaround until mainline iomap gains AOP_TRUNCATED_PAGE - * retry support. Tracks tasks that need to retry write operations due - * to DLM lock contention (-EAGAIN from FUSE server). - */ -struct fuse_dlm_retry { - bool retry_needed; -}; - /** * A Fuse connection. * @@ -1085,13 +1074,6 @@ struct fuse_conn { /* The foffset alignment in PAGE */ unsigned int alignment_pages; - - /** - * XArray tracking tasks that need DLM retry. - * Maps task pointer -> struct fuse_dlm_retry. - * Temporary workaround for iomap write deadlock. - */ - struct xarray dlm_retry_tasks; }; /* diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 7e573b4e7e4004..c8377a2868d0aa 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1523,7 +1523,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, /* module option for now */ fc->compound_open_getattr = enable_compound; - xa_init(&fc->dlm_retry_tasks); atomic64_set(&fc->attr_version, 1); atomic64_set(&fc->evict_ctr, 1); get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); @@ -1573,7 +1572,6 @@ void fuse_conn_put(struct fuse_conn *fc) } if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) fuse_backing_files_free(fc); - xa_destroy(&fc->dlm_retry_tasks); call_rcu(&fc->rcu, delayed_release); } } From 660b67b46c9c77cdfdcd7e5157cd79e5f9b91d5e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 09:10:29 +0200 Subject: [PATCH 05/17] fuse: retry a DLM grant the server refuses as contended A server answers a contended range with -EDEADLK, which fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE so the read is retried. __fuse_get_dlm_lock() returned it instead, and its writeback caller has no way to hold on to a folio it reports an error for: iomap takes the dirty flag off before offering the folio and clears its dirty ranges whatever the callback returns, so the bytes are dropped without ever reaching the server. A NOTIFY invalidate then drops the now clean folio and the next read gets the pre-write content. Ask again on -EDEADLK and -EAGAIN, on the retry budget the killed-in- flight grant already uses. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index b2a47fda316b40..62f2a9a8ff69e2 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -33,7 +33,8 @@ /* * How often to ask again for a grant a revoke killed while it was in - * flight, before giving up on the range. Each pass is a round trip. + * flight, or the server refused as contended, before giving up on the + * range. Each pass is a round trip. */ #define FUSE_DLM_GRANT_RETRIES 16 @@ -631,7 +632,17 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, if (err == -ENOSYS) { /* fuse server does not support dlm, save the info */ fc->dlm = 0; + return err; } + /* + * The range is contended, the same answer a READ gets and + * fuse_do_readfolio() turns into AOP_TRUNCATED_PAGE for its + * caller to retry. There is no such convention here, and + * the writeback caller loses the folio it is holding on an + * error, so ask again instead of reporting it. + */ + if (err == -EDEADLK || err == -EAGAIN) + goto retry; return err; } @@ -682,18 +693,8 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, * request was in flight, so the grant is dead. Retry * rather than fail: no one else holds the range, and the * write path turns an error into a failed write. - * - * Not forever, though. Every pass is a whole round trip, - * which throttles the loop but does not end it, and - * writeback asks for a grant with a folio locked, so a node - * revoking as fast as the grants arrive would hold that - * folio and this task for as long as it kept going. */ - if (fatal_signal_pending(current)) - return -EINTR; - if (!tries--) - return -EIO; - goto restart; + goto retry; } /* @@ -708,6 +709,20 @@ static int __fuse_get_dlm_lock(struct fuse_file *ff, struct inode *inode, return FUSE_DLM_GRANT_UNRECORDED; return 0; + +retry: + /* + * Ask again, but not forever. Every pass is a whole round trip, + * which throttles the loop but does not end it, and writeback asks + * for a grant with a folio locked, so a node taking the range as + * fast as this asks for it would hold that folio and this task for + * as long as it kept going. + */ + if (fatal_signal_pending(current)) + return -EINTR; + if (!tries--) + return -EIO; + goto restart; } int fuse_get_dlm_lock(struct file *file, loff_t offset, From 6fa095c479ffcc2a24fa938567707c61fdbb9ad1 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 09:11:24 +0200 Subject: [PATCH 06/17] fuse: keep a folio writeback could not send iomap clears the dirty flag before it offers a folio to ->writeback_range and clears its dirty ranges whatever the callback returns, and nothing puts either back. A run that reports an error has therefore dropped its bytes: the folio stays in the page cache, clean and uptodate, holding data the server never received, until an invalidate drops it and the next read returns what the server has. The comment claimed the opposite. Redirty the folio instead, for the grant and the allocation, while there is still a connection to send it on. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 4aa884e411e349..40e8b824924d7a 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3065,6 +3065,31 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, return false; } +/* + * Put a folio writeback could not send back on the dirty list. + * + * iomap takes the dirty flag off a folio before it offers it to + * ->writeback_range and clears its dirty ranges whatever that returns, so + * a run reporting an error has thrown its bytes away. There are no dirty + * ranges to restore: a writeback connection is refused unless the block is + * a page, so a folio holds one block and carries no iomap_folio_state. + * + * Only while there is a connection left to take them. After an abort + * every send fails, and a folio redirtied for a retry that can no longer + * happen would keep sync() going forever. + */ +static void fuse_writeback_redirty(struct fuse_conn *fc, + struct writeback_control *wbc, + struct folio *folio) +{ + if (!READ_ONCE(fc->connected)) + return; + + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); +} + static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct folio *folio, u64 pos, unsigned len, u64 end_pos) @@ -3105,15 +3130,17 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, * * fuse_dlm_regrant_range() takes the range back when it has gone, * and walks the record once under the lock held for read when it - * has not. A failure leaves the folio dirty, so the next writeback - * tries again; only a hard error stops it. + * has not. A failure redirties the folio, so the next writeback + * tries again. */ if (fc->dlm && fc->writeback_cache) { int err = fuse_dlm_regrant_range(data->ff, inode, pos, pos + len - 1); - if (err < 0 && err != -ENOSYS) + if (err < 0 && err != -ENOSYS) { + fuse_writeback_redirty(fc, wpc->wbc, folio); return err; + } } offset = offset_in_folio(folio, pos); @@ -3126,8 +3153,10 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (data->wpa == NULL) { wpa = fuse_writepage_args_setup(folio, offset, data->ff); - if (!wpa) + if (!wpa) { + fuse_writeback_redirty(fc, wpc->wbc, folio); return -ENOMEM; + } fuse_file_get(wpa->ia.ff); data->max_folios = 1; ap = &wpa->ia.ap; From 07d9e7bd34d2c28d5a2277141b498610c4d790f6 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 10:21:33 +0200 Subject: [PATCH 07/17] fuse: keep the local size against an attribute reply already on the wire The DLM buffered write holds i_rwsem shared and claims its i_size extension before it dirties anything. fuse_attr_cache_mask() decides whether the server's size wins from an i_size it reads before that claim, and sleeps in the lock tree query before fuse_change_attributes_i() applies the answer under fi->lock. A GETATTR that left while i_size still matched the server's is therefore applied over every claim made since, and i_size drops by exactly the writes in flight. truncate_pagecache() then zeroes the tail of the page holding the new size and drops what is above, which writeback sends as zeros. Move attr_version on the claim so those replies are dropped, the same reason fuse_write_update_attr() moves it, and count the writers whose claim is not yet dirty for a reply that leaves after one. FUSE_I_SIZE_UNSTABLE cannot serve as the count: it is a single bit and every writer clears it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 23 +++++++++++++++++++++++ fs/fuse/fuse_i.h | 14 ++++++++++++++ fs/fuse/inode.c | 9 ++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 40e8b824924d7a..d1d4170d35d766 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2231,6 +2231,27 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) spin_lock(&fi->lock); orig_size = i_size_read(inode); if (end > orig_size) { + /* + * Retire the attribute replies already on the + * wire. fuse_attr_cache_mask() decides whether + * the server's size wins from an i_size it read + * before this claim and before it slept in the + * lock tree query, so a GETATTR that left while + * i_size still matched the server's is applied + * afterwards and shrinks it back. Moving + * attr_version makes fuse_change_attributes_i() + * drop those replies, which is what + * fuse_write_update_attr() moves it for. + */ + fi->attr_version = + atomic64_inc_return(&fc->attr_version); + /* + * And count the claim, for a reply that leaves + * after it: until the bytes are dirtied, + * [orig_size, end) is covered by nothing else + * fuse_attr_cache_mask() can see. + */ + atomic_inc(&fi->size_extenders); i_size_write(inode, end); extended = true; } @@ -2275,6 +2296,7 @@ static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) i_size_write(inode, reached); spin_unlock(&fi->lock); } + atomic_dec(&fi->size_extenders); } if (written < 0) { @@ -4239,6 +4261,7 @@ void fuse_init_file_inode(struct inode *inode, unsigned int flags) init_waitqueue_head(&fi->direct_io_waitq); fi->notify_stamp = jiffies; fi->notify_interval_ewma = FUSE_NOTIFY_EWMA_SEED << FUSE_NOTIFY_EWMA_SHIFT; + atomic_set(&fi->size_extenders, 0); if (IS_ENABLED(CONFIG_FUSE_DAX)) fuse_dax_inode_init(inode, flags); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 6dc2595b2f7450..6981890341fdf0 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -214,6 +214,20 @@ struct fuse_inode { */ unsigned long notify_stamp; unsigned int notify_interval_ewma; + + /* + * Buffered writes that have claimed an i_size + * extension and not yet dirtied it. + * + * The DLM path holds i_rwsem shared, so several + * writers extend i_size at once and each one is + * ahead of the server until its bytes are sent. + * While this is non zero the local size wins over + * the server's; see fuse_attr_cache_mask(). + * FUSE_I_SIZE_UNSTABLE cannot serve: it is one bit + * and every writer clears it. + */ + atomic_t size_extenders; }; /* readdir cache (directory only) */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index c8377a2868d0aa..1833867ef87c24 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -569,9 +569,16 @@ static u32 fuse_attr_cache_mask(struct inode *inode, struct fuse_attr *attr, * truncate them away before they are ever sent. The grant check * alone misses them, because a page-mkwrite grant is never * recorded and a local truncate revokes its own tail grants. + * + * Both miss a write that has claimed its extension and not yet + * dirtied it: nothing is dirty there, and a NOTIFY can revoke the + * grant in between. With i_rwsem held shared several writers sit + * in that window at once, which is why they are counted rather + * than flagged. */ if (have_size && size > (loff_t) attr->size && - (fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, + (atomic_read(&fi->size_extenders) || + fuse_dlm_lock_is_held(fi, attr->size, size - attr->size, FUSE_PAGE_LOCK_WRITE) || filemap_range_needs_writeback(inode->i_mapping, attr->size, size - 1))) From a4411a030fd162cff77f1db608606f3236b85f2c Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 10:48:48 +0200 Subject: [PATCH 08/17] fuse: do not ask for a grant from inside the revoke handler fuse_reverse_inval_inode() drives this inode's page cache, before the revoke to flush it and after the revoke to drop it, on the thread that wrote the NOTIFY into /dev/fuse. Both reach fuse_iomap_writeback_range(), which holds the folio locked and under writeback while fuse_dlm_regrant_range() sends FUSE_DLM_WB_LOCK and waits. The range is the one being revoked, so the server cannot answer until the revoke completes, and the revoke cannot complete until the handler returns. A folio dirtied under a grant a NOTIFY took away is reached this way, which fuse_cache_write_iter() says is allowed to happen. Mark the task across that work and report the run as a hole when the grant is not already held: the folio goes back on the dirty list and an ordinary writeback sends it with a grant of its own. Reporting a hole means the run needs the type reset that had no user before. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 39 ++++++++++++++++++++++++++++++++++++--- fs/fuse/fuse_i.h | 30 ++++++++++++++++++++++++++++++ fs/fuse/inode.c | 12 ++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index d1d4170d35d766..0c00962b2e2b45 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3153,12 +3153,45 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, * fuse_dlm_regrant_range() takes the range back when it has gone, * and walks the record once under the lock held for read when it * has not. A failure redirties the folio, so the next writeback - * tries again. + * tries again. A revoke handler driving this skips the run + * entirely rather than ask for the grant it is taking away. */ if (fc->dlm && fc->writeback_cache) { - int err = fuse_dlm_regrant_range(data->ff, inode, pos, - pos + len - 1); + int err; + + /* + * wpc->iomap.type carries over from the previous run and from + * the previous folio, so anything that is queued has to say + * so. Left at a stale IOMAP_HOLE, iomap takes the folio for + * one it never queued and ends its writeback while the write + * is still in flight. + */ + wpc->iomap.type = IOMAP_MAPPED; + + /* + * Driven by a NOTIFY invalidate. A grant this run does not + * already hold would have to be asked for from inside the + * handler the server is waiting on, for the range that + * handler is revoking, with this folio locked and under + * writeback. The server cannot answer that until the revoke + * completes, and the revoke cannot complete until this + * returns. + * + * Report the run as a hole instead. The folio goes back on + * the dirty list and an ordinary writeback sends it with a + * grant of its own; nothing is lost and no error is recorded + * for a later fsync to report. + */ + if (fuse_in_notify_ctx() && + !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_writeback_redirty(fc, wpc->wbc, folio); + wpc->iomap.type = IOMAP_HOLE; + return len; + } + err = fuse_dlm_regrant_range(data->ff, inode, pos, + pos + len - 1); if (err < 0 && err != -ENOSYS) { fuse_writeback_redirty(fc, wpc->wbc, folio); return err; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 6981890341fdf0..cef11f9cb170c7 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,8 +32,38 @@ #include #include #include +#include #include "fuse_dlm_cache.h" +/* + * Page cache work driven by a NOTIFY invalidate is marked on the task. + * + * Writeback reached from there must not ask the server for a grant: the + * range is the one the server is revoking, and the request would go out + * from inside the handler the server is waiting on, with the folio locked + * and under writeback. See fuse_reverse_inval_inode() and + * fuse_iomap_writeback_range(). + */ +extern const char fuse_notify_ctx_key[]; + +static inline void *fuse_notify_ctx_enter(void) +{ + void *old = current->journal_info; + + current->journal_info = (void *)fuse_notify_ctx_key; + return old; +} + +static inline void fuse_notify_ctx_leave(void *old) +{ + current->journal_info = old; +} + +static inline bool fuse_in_notify_ctx(void) +{ + return current->journal_info == (void *)fuse_notify_ctx_key; +} + /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 1833867ef87c24..391c2cf432c341 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -892,6 +892,9 @@ static bool fuse_notify_inval_hot(struct fuse_inode *fi) return avg < FUSE_NOTIFY_DIO_INTERVAL; } +/* Address only; see fuse_notify_ctx_enter() */ +const char fuse_notify_ctx_key[1]; + /* * Revoke the DLM grants backing an invalidated byte range. Grants are * recorded page-aligned, so widen the revoke to page boundaries: dropping @@ -986,6 +989,14 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, forget_all_cached_acls(inode); security_inode_invalidate_secctx(inode); if (offset >= 0) { + /* + * Everything below drives this inode's page cache on behalf + * of the revoke, so mark the task: writeback reached from + * here must not send a DLM request. See + * fuse_iomap_writeback_range(). + */ + void *notify_ctx = fuse_notify_ctx_enter(); + pg_start = offset >> PAGE_SHIFT; if (len <= 0) pg_end = -1; @@ -1124,6 +1135,7 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_notify_invalidate_range(inode, pg_start, pg_end, true); } + fuse_notify_ctx_leave(notify_ctx); } iput(inode); return 0; From 73e2ae1015ad7aa64531723f9f80eff8dfd333cd Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 11:11:53 +0200 Subject: [PATCH 09/17] fuse: take the writeback grant with no folio held fuse_iomap_writeback_range() runs with the folio locked and, since iomap_writeback_folio() starts the writeback before the run loop, under writeback as well. Asking the server for a grant there is the ordering Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE-reason.txt exists to forbid: no cluster lock may be taken while a page lock is held. The read path has AOP_TRUNCATED_PAGE to unlock and retry with, and ->writeback_range has nothing of the sort, so the violation sits where the remedy does not reach. Skip a run whose grant has gone: report it as a hole, put the folio back on the dirty list and remember the range. fuse_iomap_writeback_submit() takes it back once the pass has let go of every folio, and the pass that follows sends it. Not from fuse_launder_folio(), which arrives with the folio locked by folio_unmap_invalidate(), nor from a revoke handler, which would ask for the range it is revoking. There the skip stands. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 61 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 0c00962b2e2b45..e0ff8851b1d84f 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2989,6 +2989,13 @@ struct fuse_fill_wb_data { * of U16_MAX). */ unsigned int nr_bytes; + /* + * The runs this pass could not send because their grant had gone. + * Taken back in fuse_iomap_writeback_submit(), where no folio is + * held, for the pass that follows; see fuse_iomap_writeback_range(). + */ + u64 regrant_start; + u64 regrant_end; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, @@ -3169,27 +3176,41 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, wpc->iomap.type = IOMAP_MAPPED; /* - * Driven by a NOTIFY invalidate. A grant this run does not - * already hold would have to be asked for from inside the - * handler the server is waiting on, for the range that - * handler is revoking, with this folio locked and under - * writeback. The server cannot answer that until the revoke - * completes, and the revoke cannot complete until this - * returns. + * The folio is locked and under writeback here, so a grant + * this run does not already hold must not be asked for: + * Documentation/filesystems/fuse/fuse-AOP_TRUNCATED_PAGE- + * reason.txt states the rule the read path is built around, + * that no cluster lock may be taken while a page lock is + * held. read_folio() has AOP_TRUNCATED_PAGE to unlock and + * retry with; ->writeback_range has nothing of the sort. * * Report the run as a hole instead. The folio goes back on - * the dirty list and an ordinary writeback sends it with a - * grant of its own; nothing is lost and no error is recorded - * for a later fsync to report. + * the dirty list, the range is remembered for + * fuse_iomap_writeback_submit() to take back with no folio + * held, and the pass that follows sends it. Nothing is lost + * and no error is recorded for a later fsync to report. */ - if (fuse_in_notify_ctx() && - !fuse_dlm_lock_is_held(fi, pos, len, + if (!fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) { fuse_writeback_redirty(fc, wpc->wbc, folio); + if (data->regrant_end <= data->regrant_start) { + data->regrant_start = pos; + data->regrant_end = pos + len; + } else { + data->regrant_start = min(data->regrant_start, + pos); + data->regrant_end = max(data->regrant_end, + pos + len); + } wpc->iomap.type = IOMAP_HOLE; return len; } + /* + * Held: this walks the record and sends nothing. It stays a + * call rather than the check above so a grant that arrives + * between them is still used. + */ err = fuse_dlm_regrant_range(data->ff, inode, pos, pos + len - 1); if (err < 0 && err != -ENOSYS) { @@ -3253,6 +3274,22 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, fuse_writepages_send(wpc->inode, data); } + /* + * Take back what the runs above had to skip, so the pass that + * follows finds the grant and sends the folios they left dirty. + * + * Only on the ->writepages path, which is what @wbc marks. + * fuse_launder_folio() reaches here with the folio still locked by + * folio_unmap_invalidate(), which is the ordering this is avoiding, + * and a revoke handler would ask for the very range it is revoking. + * In both the skip simply stands and writeback picks it up. + */ + if (wpc->wbc && data->ff && data->regrant_end > data->regrant_start && + !fuse_in_notify_ctx()) + fuse_dlm_regrant_range(data->ff, wpc->inode, + data->regrant_start, + data->regrant_end - 1); + if (data->ff) fuse_file_put(data->ff, false); From a04d9fb8babbb5a07e42a7e9c1152f1965569517 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 11:49:48 +0200 Subject: [PATCH 10/17] fuse: do not record a grant past what a revoke could mark fuse_dlm_request_begin() publishes the range asked for, and fuse_dlm_kill_pending() can only test that one, but the commit records what the server granted, which may reach FUSE_DLM_MAX_EXTRA_GRANT either side of it. A revoke processed while the request was on the wire and landing in that excess marks nothing: the request does not overlap it, and the shards hold no bit for it yet because the grant is not recorded. The excess is then recorded over the revoked range and no later NOTIFY takes it back, which is the case the pending list exists to catch. Publish the widest bounds the commit could record as well, and separate the two outcomes. A revoke over the range asked for still kills the grant. One over the excess alone leaves the range asked for recorded and drops the excess, which only costs a re-request. Signed-off-by: Horst Birthelmer --- fs/fuse/fuse_dlm_cache.c | 51 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index 62f2a9a8ff69e2..a456f6efee6469 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -51,13 +51,25 @@ */ #define FUSE_DLM_MAX_EXTRA_GRANT (1ULL << 30) -/* A FUSE_DLM_WB_LOCK request in flight, on cache->pending */ +/* + * A FUSE_DLM_WB_LOCK request in flight, on cache->pending. + * + * Two ranges, because the range asked for and the range that may end up + * recorded are not the same one: the server may grant more, up to + * FUSE_DLM_MAX_EXTRA_GRANT either side. A revoke has to be tested against + * both, and means something different for each. + */ struct fuse_dlm_range { /* The range asked for, as byte offsets, both inclusive */ uint64_t start; uint64_t end; - /* A revoke overlapped this request in flight */ + /* The widest [start, end] fuse_dlm_request_commit() could record */ + uint64_t wide_start; + uint64_t wide_end; + /* A revoke overlapped the range asked for: the grant is dead */ bool killed; + /* A revoke overlapped only the excess: record the asked for range */ + bool clamp; /* The cache->pending link */ struct list_head list; }; @@ -134,9 +146,12 @@ static void fuse_dlm_kill_pending(struct fuse_dlm_cache *cache, struct fuse_dlm_range *req; spin_lock(&cache->pending_lock); - list_for_each_entry(req, &cache->pending, list) + list_for_each_entry(req, &cache->pending, list) { if (req->start <= end && start <= req->end) req->killed = true; + else if (req->wide_start <= end && start <= req->wide_end) + req->clamp = true; + } spin_unlock(&cache->pending_lock); } @@ -301,7 +316,17 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, req->start = start; req->end = end; + /* + * The bounds __fuse_get_dlm_lock() caps the recorded grant to. A + * revoke between here and the commit must be seen by one of the two + * tests in fuse_dlm_kill_pending(), or it would be recorded over. + */ + req->wide_start = start > FUSE_DLM_MAX_EXTRA_GRANT ? + start - FUSE_DLM_MAX_EXTRA_GRANT : 0; + req->wide_end = U64_MAX - end < FUSE_DLM_MAX_EXTRA_GRANT ? + U64_MAX : end + FUSE_DLM_MAX_EXTRA_GRANT; req->killed = false; + req->clamp = false; spin_lock(&cache->pending_lock); list_add_tail(&req->list, &cache->pending); @@ -321,9 +346,20 @@ void fuse_dlm_request_begin(struct fuse_inode *inode, * lands either before it and is seen on @req, or after it and finds the * grant in the tree. * + * A revoke processed while @req was in flight lands in one of three + * places, and fuse_dlm_kill_pending() has already said which: + * + * - over the range asked for. The grant may predate it and there is no + * way to tell, so nothing is recorded and the caller asks again. + * - over the excess the server volunteered beyond it, and nothing else. + * The range asked for is untouched by it and is recorded; the excess + * is dropped, which only costs a re-request. + * - outside both, which says nothing about this grant. The whole of + * [start, end] is recorded. + * * @req is retired in every case and may be reused. * - * Return: -EAGAIN if a revoke overlapped @req while it was in flight, + * Return: -EAGAIN if a revoke overlapped the range @req asked for, * nothing recorded; otherwise the result of recording the grant. */ int fuse_dlm_request_commit(struct fuse_inode *inode, @@ -331,7 +367,7 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, uint64_t end, enum fuse_page_lock_mode mode) { struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; - bool revoked; + bool revoked, clamp; int ret = 0; /* @@ -344,6 +380,11 @@ int fuse_dlm_request_commit(struct fuse_inode *inode, spin_lock(&cache->pending_lock); list_del(&req->list); revoked = req->killed; + clamp = req->clamp; + if (clamp) { + start = req->start; + end = req->end; + } spin_unlock(&cache->pending_lock); if (!revoked) From e7b2183db6bdc6eec6b9851019aed967c50fd66e Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 11:52:53 +0200 Subject: [PATCH 11/17] fuse: say that the readahead grant is taken under the folio locks The comment claimed the round trip happens before any folio of the window is locked. ->readahead is entered with all of them locked and readahead_folio() is what unlocks them, so it happens under them. Say so, and say why it does not close a cycle: the lock in the way belongs to the node the revoke is sent to, not to this one. Name the case that does not follow, a window this already holds part of. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e0ff8851b1d84f..57b326391f3e70 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1322,9 +1322,23 @@ static void fuse_readahead(struct readahead_control *rac) * left in @rac. A server without DLM support answers -ENOSYS and * clears fc->dlm, which is not a failure. * - * The round trip is taken before any folio of the window is locked - * and with nothing fenced out, so it holds up this reader and - * nothing else. + * ->readahead is entered with every folio of the window already + * locked, and pulling one off @rac is what unlocks it, so the round + * trip is taken under those locks. See the readahead line of + * Documentation/filesystems/locking.rst. + * + * What keeps that from closing a cycle is the direction a revoke + * travels. This asks for a read grant on a range it holds nothing + * on, so the lock in the way is another node's, and the revoke that + * frees it is sent there. Nothing here has to run for this request + * to be answered, so the folios stay locked only for as long as the + * round trip. + * + * A window this already holds part of is the open edge: the query + * fails for the whole of it, so the request goes out while that part + * is still held, and a revoke for that part is sent here. Whether + * the server can answer while a revoke of its own is outstanding is + * not something this side can know. */ if (fc->writeback_cache && fc->dlm) { int err = fuse_get_dlm_lock(rac->file, readahead_pos(rac), From 20e357a145bbf7a5e3b4da1fbef2e28c5b382eb6 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 13:30:27 +0200 Subject: [PATCH 12/17] fuse: bound the range a writeback pass defers fuse_iomap_writeback_range() unions every run it had to skip, and fuse_iomap_writeback_submit() asks for the lot in one grant. A pass sweeping a large file skips runs gigabytes apart, so the union grows to the whole sweep and the request covers a range nothing wanted. Stop extending at one shard. The runs left out stay dirty and a later pass asks for them. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 57b326391f3e70..1231fc4e640775 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3211,10 +3211,21 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, data->regrant_start = pos; data->regrant_end = pos + len; } else { - data->regrant_start = min(data->regrant_start, - pos); - data->regrant_end = max(data->regrant_end, - pos + len); + u64 s = min(data->regrant_start, pos); + u64 e = max(data->regrant_end, pos + len); + + /* + * One shard is as far as a single request is + * worth taking back. A pass sweeping a large + * file skips runs a long way apart, and the + * span between them says nothing about what + * is wanted; the runs left out stay dirty and + * a later pass asks for them. + */ + if (e - s <= FUSE_DLM_SHARD_SIZE) { + data->regrant_start = s; + data->regrant_end = e; + } } wpc->iomap.type = IOMAP_HOLE; return len; From d444eeec5bdd44bbf2b35dedd1eef3aeed8f9dd3 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 13:43:16 +0200 Subject: [PATCH 13/17] fuse: put a skipped folio back on the dirty list after iomap lets go fuse_writeback_redirty() dirtied the folio from inside ->writeback_range, and iomap_writeback_folio() runs iomap_clear_range_dirty() over the whole folio once that returns. For a folio one block wide there is no iomap_folio_state and the call does nothing, so it worked. A large folio has one, and the folio is left with the dirty flag and no dirty block under it: the next pass finds nothing to write and the folio goes clean with its bytes never sent. Hold the folio instead and dirty it once iomap has released it, on the next call or in the submit. fuse_writepage_need_send() also reached wbc->range_end with no wbc, which only a folio of a single run kept out of reach; guard it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 68 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 1231fc4e640775..23fb0ca06313b6 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -3010,6 +3010,11 @@ struct fuse_fill_wb_data { */ u64 regrant_start; u64 regrant_end; + /* + * A folio whose run was skipped, held by a reference until it can be + * put back on the dirty list; see fuse_writeback_redirty(). + */ + struct folio *redirty; }; static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, @@ -3093,7 +3098,7 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, unsigned int total_pages = (bytes + PAGE_SIZE - 1) >> PAGE_SHIFT; pgoff_t page_index = pos >> PAGE_SHIFT; - if (!(page_index % fc->alignment_pages)) { + if (wbc && !(page_index % fc->alignment_pages)) { pgoff_t end_page_index = (wbc->range_end + PAGE_SIZE - 1) >> PAGE_SHIFT; /* we are at a point where we would write aligned @@ -3112,25 +3117,56 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, * Put a folio writeback could not send back on the dirty list. * * iomap takes the dirty flag off a folio before it offers it to - * ->writeback_range and clears its dirty ranges whatever that returns, so - * a run reporting an error has thrown its bytes away. There are no dirty - * ranges to restore: a writeback connection is refused unless the block is - * a page, so a folio holds one block and carries no iomap_folio_state. + * ->writeback_range, and a run reporting an error, or reporting a hole + * because the grant has gone, has thrown its bytes away unless they are put + * back. + * + * Not from inside the callback, though. iomap_writeback_folio() runs + * iomap_clear_range_dirty() over the whole folio once that has returned, so + * a range put back there is wiped again. For a folio one block wide that + * call does nothing and it would not matter, but a large folio carries an + * iomap_folio_state, and the folio would then be left with the dirty flag + * and no dirty block under it: the next pass finds nothing to write and the + * folio goes clean with its bytes never sent. + * + * So hold the folio and dirty it once iomap has let go of it, which on the + * ->writepages path is the next call or the submit, and in + * fuse_launder_folio() is the submit it makes itself. * - * Only while there is a connection left to take them. After an abort + * Only while there is a connection left to take the bytes. After an abort * every send fails, and a folio redirtied for a retry that can no longer * happen would keep sync() going forever. */ +static void fuse_writeback_redirty_done(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, + struct writeback_control *wbc) +{ + struct folio *folio = data->redirty; + + if (!folio) + return; + data->redirty = NULL; + + if (READ_ONCE(fc->connected)) { + folio_mark_dirty(folio); + if (wbc) + wbc->pages_skipped += folio_nr_pages(folio); + } + folio_put(folio); +} + +/* Remember @folio for fuse_writeback_redirty_done() */ static void fuse_writeback_redirty(struct fuse_conn *fc, + struct fuse_fill_wb_data *data, struct writeback_control *wbc, struct folio *folio) { - if (!READ_ONCE(fc->connected)) + if (data->redirty == folio) return; - folio_mark_dirty(folio); - if (wbc) - wbc->pages_skipped += folio_nr_pages(folio); + fuse_writeback_redirty_done(fc, data, wbc); + folio_get(folio); + data->redirty = folio; } static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, @@ -3161,6 +3197,11 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, fuse_wb_token_put(data->wb_token); data->wb_token = NULL; data->wb_folio = folio; + /* + * iomap has unlocked whatever it offered before this, so a + * folio held from then can go back on the dirty list now. + */ + fuse_writeback_redirty_done(fc, data, wpc->wbc); } /* @@ -3206,7 +3247,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, */ if (!fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) { - fuse_writeback_redirty(fc, wpc->wbc, folio); + fuse_writeback_redirty(fc, data, wpc->wbc, folio); if (data->regrant_end <= data->regrant_start) { data->regrant_start = pos; data->regrant_end = pos + len; @@ -3239,7 +3280,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, err = fuse_dlm_regrant_range(data->ff, inode, pos, pos + len - 1); if (err < 0 && err != -ENOSYS) { - fuse_writeback_redirty(fc, wpc->wbc, folio); + fuse_writeback_redirty(fc, data, wpc->wbc, folio); return err; } } @@ -3255,7 +3296,7 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (data->wpa == NULL) { wpa = fuse_writepage_args_setup(folio, offset, data->ff); if (!wpa) { - fuse_writeback_redirty(fc, wpc->wbc, folio); + fuse_writeback_redirty(fc, data, wpc->wbc, folio); return -ENOMEM; } fuse_file_get(wpa->ia.ff); @@ -3293,6 +3334,7 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, fuse_wb_token_put(data->wb_token); data->wb_token = NULL; data->wb_folio = NULL; + fuse_writeback_redirty_done(get_fuse_conn(wpc->inode), data, wpc->wbc); if (data->wpa) { WARN_ON(!data->wpa->ia.ap.num_folios); From dd2e0db7b16f2bd06fdd00c2ab0e05a70eee7cc0 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 13:53:10 +0200 Subject: [PATCH 14/17] fuse: fix the two writethrough paths a large folio breaks fuse_fill_write_pages() carried the offset of the next copy across iterations as a within page residue and turned it into a folio offset by adding the page delta. A write that starts inside a folio it does not begin covers the rest of that folio, and the residue left from it is then added to the next folio's own delta: the copy lands past the end of the folio, copy_folio_from_iter_atomic() warns and returns nothing, and the fault in loop retries it for good. A folio one page wide cannot start before the write does, which is what kept it out of reach. Take the offset from @pos, which is always current, and read the loop's carry on test off the folio it just filled. fuse_page_mkwrite() locked the page that faulted. The whole folio is dirtied on the way out, in fault_dirty_shared_page(), so the DLM lock has to cover the folio. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 23fb0ca06313b6..31f6c5fa609fba 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1614,7 +1614,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, { struct fuse_args_pages *ap = &ia->ap; struct fuse_conn *fc = get_fuse_conn(mapping->host); - unsigned offset = pos & (PAGE_SIZE - 1); size_t count = 0; unsigned int num; int err = 0; @@ -1641,7 +1640,13 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, if (mapping_writably_mapped(mapping)) flush_dcache_folio(folio); - folio_offset = ((index - folio->index) << PAGE_SHIFT) + offset; + /* + * From @pos, not carried across iterations: a write landing + * inside a folio it does not start covers the rest of it, and + * a residue kept from that lands the next folio's copy past + * its end. + */ + folio_offset = offset_in_folio(folio, pos); bytes = min(folio_size(folio) - folio_offset, num); tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); @@ -1671,9 +1676,6 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, count += tmp; pos += tmp; num -= tmp; - offset += tmp; - if (offset == folio_size(folio)) - offset = 0; /* If we copied full folio, mark it uptodate */ if (tmp == folio_size(folio)) @@ -1685,7 +1687,12 @@ static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, ia->write.folio_locked = true; break; } - if (!fc->big_writes || offset != 0) + /* + * Carry on only from a folio boundary: a copy that stopped + * short leaves the next one starting inside a folio, which is + * one request's worth on its own. + */ + if (!fc->big_writes || folio_offset + tmp != folio_size(folio)) break; } @@ -3502,9 +3509,13 @@ static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) struct fuse_mount *fm = get_fuse_mount(inode); if (fm->fc->dlm) { - loff_t pos = vmf->pgoff << PAGE_SHIFT; - size_t length = PAGE_SIZE; - int err = fuse_get_page_mkwrite_lock(file, pos, length); + /* + * The whole folio is dirtied on the way out of this fault + * (fault_dirty_shared_page()), so the lock has to cover the + * folio, not the page that faulted. + */ + int err = fuse_get_page_mkwrite_lock(file, folio_pos(folio), + folio_size(folio)); if (err < 0) { return vmf_error(err); } From a3d93d00bad6e76f8ee093606f7045e42507d73d Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 18:23:59 +0200 Subject: [PATCH 15/17] fuse: do not lose a folio writeback declined to send Two ways a cached write reached the write check as a hole. fuse_iomap_writeback_range() returns -EIO when the inode has no file open for writing, and dropped the folio doing so. fuse_open() invalidates the whole mapping unless the server sets FOPEN_KEEP_CACHE, and reaches fuse_launder_folio() that way; by the time a reader opens the file the last writer has closed, so fuse_write_file_get() finds nothing, warns, and the bytes go. They are still the newest there are, so keep them. And a run whose grant had gone is skipped, its range taken back in the submit, and the folio left dirty for a later pass. A data integrity writeback has no later pass, so fsync() and close() reported bytes written that were still only in the page cache. Go round again while anything is left deferred. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 65 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 31f6c5fa609fba..75dde9f1596220 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -2994,6 +2994,13 @@ static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio return wpa; } +/* + * How many times a data integrity writeback goes round for runs it had to + * skip. Each pass takes the grants the one before it deferred, so one more + * is normally enough; the cap is there because a revoke can take them again. + */ +#define FUSE_WB_DEFER_PASSES 4 + struct fuse_fill_wb_data { struct fuse_writepage_args *wpa; struct fuse_file *ff; @@ -3017,6 +3024,8 @@ struct fuse_fill_wb_data { */ u64 regrant_start; u64 regrant_end; + /* fuse_iomap_writeback_submit() took that range back */ + bool regranted; /* * A folio whose run was skipped, held by a reference until it can be * put back on the dirty list; see fuse_writeback_redirty(). @@ -3192,8 +3201,18 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (!data->ff) { data->ff = fuse_write_file_get(fi); - if (!data->ff) + if (!data->ff) { + /* + * No file left open for writing, which + * fuse_open()'s invalidate reaches through + * fuse_launder_folio() once the last writer has + * closed. The bytes are still the newest there + * are, so keep them: dropping them here loses a + * write that fsync and close both reported done. + */ + fuse_writeback_redirty(fc, data, wpc->wbc, folio); return -EIO; + } } /* @@ -3359,10 +3378,12 @@ static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, * In both the skip simply stands and writeback picks it up. */ if (wpc->wbc && data->ff && data->regrant_end > data->regrant_start && - !fuse_in_notify_ctx()) + !fuse_in_notify_ctx()) { fuse_dlm_regrant_range(data->ff, wpc->inode, data->regrant_start, data->regrant_end - 1); + data->regranted = true; + } if (data->ff) fuse_file_put(data->ff, false); @@ -3380,14 +3401,8 @@ static int fuse_writepages(struct address_space *mapping, { struct inode *inode = mapping->host; struct fuse_conn *fc = get_fuse_conn(inode); - struct fuse_fill_wb_data data = {}; - struct iomap_writepage_ctx wpc = { - .inode = inode, - .iomap.type = IOMAP_MAPPED, - .wbc = wbc, - .ops = &fuse_writeback_ops, - .wb_ctx = &data, - }; + unsigned int tries = FUSE_WB_DEFER_PASSES; + int err; if (fuse_is_bad(inode)) return -EIO; @@ -3396,7 +3411,35 @@ static int fuse_writepages(struct address_space *mapping, fc->num_background >= fc->congestion_threshold) return 0; - return iomap_writepages(&wpc); + /* + * A run whose grant had gone is skipped and its range taken back in + * fuse_iomap_writeback_submit(), which leaves the folio dirty for a + * later pass. For a data integrity writeback there is no later + * pass: fsync() and close() would report the bytes written while + * they are still only in the page cache. Go round again, now that + * the grant is held, until nothing is left deferred. + */ + do { + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = inode, + .iomap.type = IOMAP_MAPPED, + .wbc = wbc, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + err = iomap_writepages(&wpc); + /* + * Only where the submit took the grant back. A revoke + * handler driving this is not allowed to, so the runs it + * skipped would be skipped again by every pass. + */ + if (err || wbc->sync_mode != WB_SYNC_ALL || !data.regranted) + break; + } while (--tries); + + return err; } static int fuse_launder_folio(struct folio *folio) From 520aed9f669f60dae125c354a388330c0a867733 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Mon, 31 Aug 2026 21:26:38 +0200 Subject: [PATCH 16/17] fuse: pin a dlm grant from confirmation to dirty A writer confirms its grant and then copies and dirties, and a revoke landing in between is answered while those bytes are in no page cache and on no wire. The flush the revoke runs cannot find them, so they go out later under a fresh grant, after the server has handed the lock to another node. Add a pin: the range a writer is about to dirty, on a list a revoke walks before it removes anything. A revoke publishes the range it takes away on a second list and waits for the pins overlapping it, and a pin overlapping a published range is refused. Refusal and wait test the same overlap, so the wait converges, and ranges that do not overlap never meet: a notify leaves the rest of the file writable and a write outside the notified range does not hold it up. Both nodes are caller storage, so nothing is allocated to take a pin and the writeback path can take one with a folio held. A pin is found again by owner, since iomap hands ->put_folio the inode and nothing of the iteration. Confirm the grant under the pin, never before, in the three places bytes become the server's: the iomap interior through new ->get_folio and ->put_folio hooks, the writethrough edges around their FUSE_WRITE, and the writeback run until the folio is under writeback. A grant that has gone is asked for again with the pin dropped, since that request is answered by the server the revoke came from. With the writers fenced, the flush a revoke runs needs a single pass: nothing can turn up dirty behind it. Signed-off-by: Horst Birthelmer --- fs/fuse/file.c | 177 +++++++++++++++++++++++++++++-- fs/fuse/fuse_dlm_cache.c | 217 +++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_dlm_cache.h | 72 +++++++++++++ fs/fuse/fuse_i.h | 39 ++++++- fs/fuse/inode.c | 43 +++++++- 5 files changed, 535 insertions(+), 13 deletions(-) diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 75dde9f1596220..a0f7384448d5a8 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -1186,6 +1186,11 @@ struct fuse_iomap_write_ctx { struct file *file; /* fuse_iomap_read_folio_range() hit AOP_TRUNCATED_PAGE */ bool retry_needed; + /* + * The grant ->get_folio pinned, live until ->put_folio drops it. + * Here rather than on their stacks, which do not span the pair. + */ + struct fuse_dlm_span pin; }; static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, @@ -1848,10 +1853,105 @@ static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive, bool uncached) } } +/* + * How many times a writer confirms its grant again before giving up on + * the range. A pass costs a round trip only when the grant has gone, + * which is a revoke landing between the request and the confirmation. + */ +#define FUSE_DLM_PIN_RETRIES 16 + +/* + * Pin [@pos, @pos + @len) with the grant over it confirmed, so the bytes + * can be dirtied under a lock that cannot be taken away meanwhile; see + * fuse_dlm_pin(). @pin is the caller's storage for the pin, which it + * drops with fuse_dlm_unpin() once the bytes are dirty. + * + * The grant is asked for again when it has gone, and the pin must not be + * held across that request: it is answered by the server the revoke + * waiting for the pin came from. So confirm and request alternate, and + * no folio may be held here. + */ +static int fuse_dlm_pin_write(struct file *file, struct fuse_dlm_span *pin, + loff_t pos, size_t len) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + unsigned int tries = FUSE_DLM_PIN_RETRIES; + int err; + + for (;;) { + fuse_dlm_pin(fi, pin, pos, len); + /* + * A server that turned out to have no DLM leaves nothing to + * confirm, and the pin still pairs with the caller's unpin. + */ + if (!fc->dlm || + fuse_dlm_lock_is_held(fi, pos, len, FUSE_PAGE_LOCK_WRITE)) + return 0; + fuse_dlm_unpin(fi); + + if (!tries--) + return -EIO; + + err = fuse_get_dlm_lock(file, pos, len, FUSE_PAGE_LOCK_WRITE); + if (err < 0 && err != -ENOSYS) + return err; + if (err > 0) { + /* + * Granted but unrecorded, so there is nothing for the + * confirmation above to find. The range is covered + * cluster-wide; pin and proceed. + */ + fuse_dlm_pin(fi, pin, pos, len); + return 0; + } + } +} + +/* + * The grant over the bytes about to be copied, held until ->put_folio + * hands the folio back dirty. Nothing is locked yet, which is what lets + * this wait out a revoke and ask for the grant again; ->writeback_range + * can do neither. + */ +static struct folio *fuse_iomap_get_folio(struct iomap_iter *iter, loff_t pos, + unsigned int len) +{ + struct fuse_iomap_write_ctx *ctx = iter->private; + struct folio *folio; + int err; + + err = fuse_dlm_pin_write(ctx->file, &ctx->pin, pos, len); + if (err) + return ERR_PTR(err); + + folio = iomap_get_folio(iter, pos, len); + if (IS_ERR(folio)) + fuse_dlm_unpin(get_fuse_inode(iter->inode)); + + return folio; +} + +/* iomap_write_end() has dirtied whatever was copied by now */ +static void fuse_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int copied, struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); + fuse_dlm_unpin(get_fuse_inode(inode)); +} + static const struct iomap_write_ops fuse_iomap_write_ops = { .read_folio_range = fuse_iomap_read_folio_range, }; +static const struct iomap_write_ops fuse_iomap_dlm_write_ops = { + .get_folio = fuse_iomap_get_folio, + .put_folio = fuse_iomap_put_folio, + .read_folio_range = fuse_iomap_read_folio_range, +}; + static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) @@ -1878,6 +1978,7 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * pointer, which fuse needs for @file either way. */ struct fuse_iomap_write_ctx ctx = { .file = file }; + struct fuse_conn *fc = get_fuse_conn(file_inode(file)); ssize_t written, total_written = 0; retry: @@ -1896,7 +1997,9 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, * and granular dirty tracking for large folios. */ written = iomap_file_buffered_write(iocb, from, &fuse_iomap_ops, - &fuse_iomap_write_ops, &ctx); + fc->dlm ? &fuse_iomap_dlm_write_ops : + &fuse_iomap_write_ops, + &ctx); if (written > 0) total_written += written; @@ -1929,6 +2032,7 @@ static ssize_t fuse_writeback_write_iter(struct kiocb *iocb, static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, struct file *file, size_t len, bool through) { + struct fuse_dlm_span pin; size_t hidden; ssize_t res; @@ -1938,8 +2042,21 @@ static ssize_t fuse_dlm_write_chunk(struct kiocb *iocb, struct iov_iter *from, /* Cap the iterator to this chunk, keeping the tail for later chunks. */ hidden = iov_iter_count(from) - len; iov_iter_truncate(from, len); - res = through ? fuse_perform_write(iocb, from, true) - : fuse_writeback_write_iter(iocb, from, file); + if (through) { + /* + * These bytes never enter the page cache, so a revoke cannot + * find them by flushing it. Hold the grant across the + * FUSE_WRITE instead, which is the reply the revoke already + * waits for when the same bytes go through writeback. + */ + res = fuse_dlm_pin_write(file, &pin, iocb->ki_pos, len); + if (!res) { + res = fuse_perform_write(iocb, from, true); + fuse_dlm_unpin(get_fuse_inode(file_inode(file))); + } + } else { + res = fuse_writeback_write_iter(iocb, from, file); + } /* Restore from the iterator's own residue, so short writes/errors * (which leave it partly advanced) reexpand to the exact remainder. */ iov_iter_reexpand(from, iov_iter_count(from) + hidden); @@ -1990,7 +2107,8 @@ static ssize_t fuse_dlm_buffered_write(struct kiocb *iocb, /* No whole page inside the write: nothing cacheable, all through. */ if (mid_end <= mid_start) - return fuse_perform_write(iocb, from, true); + return fuse_dlm_write_chunk(iocb, from, file, + iov_iter_count(from), true); /* * Every chunk reports a failure the same way: the error while @@ -3195,6 +3313,8 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, struct inode *inode = wpc->inode; struct fuse_inode *fi = get_fuse_inode(inode); struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_dlm_span pin; + bool pinned = false; loff_t offset; WARN_ON_ONCE(!data); @@ -3243,6 +3363,11 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, * has not. A failure redirties the folio, so the next writeback * tries again. A revoke handler driving this skips the run * entirely rather than ask for the grant it is taking away. + * + * The grant is pinned over this run from the moment it is found + * until the folio is under writeback, where the revoke waits for it + * again. Without that these bytes could still go out under a + * revoke that has already been answered. */ if (fc->dlm && fc->writeback_cache) { int err; @@ -3256,6 +3381,18 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, */ wpc->iomap.type = IOMAP_MAPPED; + /* + * The revoke handler flushing the range it is taking + * away. That lock is still this client's until the + * handler returns, so send without asking: the record has + * gone already and asking would be a round trip for the + * very range being revoked. Only for that range, since + * the latched path launders the whole mapping and the + * rest of it may be covered by nothing. + */ + if (fuse_in_notify_range(pos, len)) + goto queue; + /* * The folio is locked and under writeback here, so a grant * this run does not already hold must not be asked for: @@ -3270,9 +3407,19 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, * fuse_iomap_writeback_submit() to take back with no folio * held, and the pass that follows sends it. Nothing is lost * and no error is recorded for a later fsync to report. + * + * A refused pin is a revoke of this range draining, and + * leaves the run in the same place for the same reason. A + * revoke elsewhere in the file does not refuse it. */ - if (!fuse_dlm_lock_is_held(fi, pos, len, - FUSE_PAGE_LOCK_WRITE)) { + pinned = fuse_dlm_trypin(fi, &pin, pos, len); + if (pinned && !fuse_dlm_lock_is_held(fi, pos, len, + FUSE_PAGE_LOCK_WRITE)) { + fuse_dlm_unpin(fi); + pinned = false; + } + + if (!pinned) { fuse_writeback_redirty(fc, data, wpc->wbc, folio); if (data->regrant_end <= data->regrant_start) { data->regrant_start = pos; @@ -3299,17 +3446,19 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, } /* - * Held: this walks the record and sends nothing. It stays a - * call rather than the check above so a grant that arrives - * between them is still used. + * Held, and pinned so it stays held: this walks the record + * and sends nothing. It stays a call rather than the check + * above so a grant that arrives between them is still used. */ err = fuse_dlm_regrant_range(data->ff, inode, pos, pos + len - 1); if (err < 0 && err != -ENOSYS) { fuse_writeback_redirty(fc, data, wpc->wbc, folio); + fuse_dlm_unpin(fi); return err; } } +queue: offset = offset_in_folio(folio, pos); @@ -3323,6 +3472,8 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, wpa = fuse_writepage_args_setup(folio, offset, data->ff); if (!wpa) { fuse_writeback_redirty(fc, data, wpc->wbc, folio); + if (pinned) + fuse_dlm_unpin(fi); return -ENOMEM; } fuse_file_get(wpa->ia.ff); @@ -3338,6 +3489,14 @@ static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, if (!data->wb_token) data->wb_token = fuse_wb_token_alloc(inode, folio); + /* + * Under writeback now, so the flush a revoke runs before it takes + * the grant away waits for these bytes. Nothing further is needed + * to keep them in front of the handover. + */ + if (pinned) + fuse_dlm_unpin(fi); + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, offset, len, data->wb_token); data->nr_bytes += len; diff --git a/fs/fuse/fuse_dlm_cache.c b/fs/fuse/fuse_dlm_cache.c index a456f6efee6469..d3ef8548d3b380 100644 --- a/fs/fuse/fuse_dlm_cache.c +++ b/fs/fuse/fuse_dlm_cache.c @@ -167,6 +167,223 @@ void fuse_dlm_cache_init(struct fuse_inode *inode) xa_init(&cache->shards); spin_lock_init(&cache->pending_lock); INIT_LIST_HEAD(&cache->pending); + spin_lock_init(&cache->pin_lock); + INIT_LIST_HEAD(&cache->pins); + INIT_LIST_HEAD(&cache->fences); + init_waitqueue_head(&cache->pin_wq); +} + +/* + * Set up @span over the page-aligned range [@offset, @offset + @length), + * the same range a fuse_dlm_lock_is_held() with these arguments asks + * about, so a fence over a page cannot miss a pin on that page. + */ +static void fuse_dlm_span_set(struct fuse_dlm_span *span, loff_t offset, + size_t length) +{ + span->start = (uint64_t)offset & PAGE_MASK; + span->end = ((uint64_t)offset + length - 1) | (PAGE_SIZE - 1); + span->owner = current; +} + +/* + * Does anything on @head share a byte with [@start, @end]? Caller holds + * fuse_dlm_cache.pin_lock. + */ +static bool fuse_dlm_overlaps_locked(struct list_head *head, uint64_t start, + uint64_t end) +{ + struct fuse_dlm_span *span; + + list_for_each_entry(span, head, list) + if (span->start <= end && start <= span->end) + return true; + + return false; +} + +/* fuse_dlm_overlaps_locked() taking the lock itself */ +static bool fuse_dlm_overlaps(struct fuse_dlm_cache *cache, + struct list_head *head, uint64_t start, + uint64_t end) +{ + bool overlap; + + spin_lock(&cache->pin_lock); + overlap = fuse_dlm_overlaps_locked(head, start, end); + spin_unlock(&cache->pin_lock); + + return overlap; +} + +/** + * fuse_dlm_pin - hold the grants over a range until fuse_dlm_unpin() + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * Waits out a revoke overlapping that range, so it must not be called + * with a folio held: the revoke drops that same page cache once it has + * drained. A revoke elsewhere in the file is not waited for. The caller + * confirms its grant after this returns, never before; a confirmation + * from before the pin says nothing. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + /* + * A revoke handler driving this inode's page cache: the lock over + * the range is still this client's until the handler returns, and + * the fence this would wait on is the handler's own. Both ends + * test the same task, so nothing is left on the list. + */ + if (fuse_in_notify_ctx()) + return; + + fuse_dlm_span_set(pin, offset, length); + + spin_lock(&cache->pin_lock); + while (fuse_dlm_overlaps_locked(&cache->fences, pin->start, pin->end)) { + spin_unlock(&cache->pin_lock); + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->fences, + pin->start, pin->end)); + spin_lock(&cache->pin_lock); + } + /* + * At the head, so fuse_dlm_unpin() drops the innermost pin of a + * task that holds more than one. + */ + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); +} + +/** + * fuse_dlm_trypin - fuse_dlm_pin() for a caller that cannot sleep + * @inode: the fuse inode + * @pin: caller-owned storage, live until the unpin + * @offset: byte offset the caller is about to write + * @length: length of the region in bytes + * + * For the writeback path, which holds a folio locked and under writeback + * and has nothing to wait with. A refusal means a revoke of this range + * is draining; the caller redirties and the pass that follows sends the + * folio. + * + * Return: true if the range is pinned, false if it is not. + */ +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + bool fenced; + + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return true; + + fuse_dlm_span_set(pin, offset, length); + + spin_lock(&cache->pin_lock); + fenced = fuse_dlm_overlaps_locked(&cache->fences, pin->start, + pin->end); + if (!fenced) + list_add(&pin->list, &cache->pins); + spin_unlock(&cache->pin_lock); + + return !fenced; +} + +/** + * fuse_dlm_unpin - release the pin this task last took on @inode + * @inode: the fuse inode + * + * Found by owner rather than by node: iomap hands ->put_folio the inode + * and nothing of the iteration, and a task holds one pin at a time. + */ +void fuse_dlm_unpin(struct fuse_inode *inode) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + struct fuse_dlm_span *pin; + bool waiters; + + /* See fuse_dlm_pin() */ + if (fuse_in_notify_ctx()) + return; + + spin_lock(&cache->pin_lock); + list_for_each_entry(pin, &cache->pins, list) { + if (pin->owner == current) { + list_del(&pin->list); + break; + } + } + waiters = !list_empty(&cache->fences); + spin_unlock(&cache->pin_lock); + + if (waiters) + wake_up_all(&cache->pin_wq); +} + +/** + * fuse_dlm_revoke_begin - fence the writers over a range that have not + * dirtied yet + * @inode: the fuse inode + * @fence: caller-owned storage, live until fuse_dlm_revoke_end() + * @offset: start byte offset being revoked + * @len: length in bytes, or <= 0 for everything from @offset on + * + * Publishes the range, then waits for the pins over it taken before it. + * On return no thread is between confirming a grant on that range and + * dirtying under it, and none can start, so what the caller flushes is + * everything the grants it is about to drop can have produced. A writer + * elsewhere in the file is neither waited for nor held up. + * + * Publishing before waiting is what makes the wait converge: a pin is + * refused on the same overlap this waits on, so nothing admitted after + * this can prolong it. + * + * Revokes on one inode fence independently, each over its own range. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + /* The range fuse_dlm_unlock_range() will be asked to drop */ + fence->start = (uint64_t)offset & PAGE_MASK; + fence->end = len <= 0 ? U64_MAX : + (((uint64_t)offset + len - 1) | (PAGE_SIZE - 1)); + fence->owner = NULL; + + spin_lock(&cache->pin_lock); + list_add(&fence->list, &cache->fences); + spin_unlock(&cache->pin_lock); + + wait_event(cache->pin_wq, + !fuse_dlm_overlaps(cache, &cache->pins, fence->start, + fence->end)); +} + +/** + * fuse_dlm_revoke_end - drop the fence fuse_dlm_revoke_begin() published + * @inode: the fuse inode + * @fence: the fence published there + */ +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence) +{ + struct fuse_dlm_cache *cache = &inode->dlm_locked_areas; + + spin_lock(&cache->pin_lock); + list_del(&fence->list); + spin_unlock(&cache->pin_lock); + + wake_up_all(&cache->pin_wq); } /** diff --git a/fs/fuse/fuse_dlm_cache.h b/fs/fuse/fuse_dlm_cache.h index c54cad9d000226..ec2e82c406a1cb 100644 --- a/fs/fuse/fuse_dlm_cache.h +++ b/fs/fuse/fuse_dlm_cache.h @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -22,6 +23,21 @@ struct fuse_file; /* Lock modes for page ranges */ enum fuse_page_lock_mode { FUSE_PAGE_LOCK_READ, FUSE_PAGE_LOCK_WRITE }; +/* + * A range held on one of the two lists in struct fuse_dlm_cache: a + * writer between confirming a grant and dirtying under it (@owner set), + * or a revoke taking grants away (@owner NULL). Caller-owned storage, + * live until the matching unpin or revoke end. + */ +struct fuse_dlm_span { + /* Page-aligned byte offsets, both inclusive */ + uint64_t start; + uint64_t end; + /* The pinning task, NULL for a revoke */ + struct task_struct *owner; + struct list_head list; +}; + /* * fuse_get_dlm_lock() result: the server granted the lock but recording * it locally failed, leaving the grant invisible to @@ -98,6 +114,22 @@ struct fuse_dlm_shard { * @pending_lock the pending list. Innermost, and the only lock * fuse_dlm_request_begin() and fuse_dlm_request_abort() * take at all. + * @pin_lock the pin and fence lists. Innermost, taken alone, and + * never held across a sleep. + * + * @lock says what is covered now, which is not enough for a writer: it + * confirms a grant, then copies and dirties, and a revoke landing in + * between sends those bytes out after the server has handed the lock on. + * The pin closes that: a revoke waits for the pins over the range it is + * taking away before it removes anything, so a grant confirmed under a + * pin is still held when the bytes become visible to writeback. + * + * Both sides are ranges rather than a count, so a revoke fences only the + * writers it overlaps and a write outside it runs on. Refusal and wait + * test the same overlap, which is what makes the wait converge: once a + * fence is published no pin that would prolong it is admitted. The + * nodes are caller storage, so nothing is allocated to take a pin and + * the writeback path can take one with a folio held. */ struct fuse_dlm_cache { /* See the locking comment above */ @@ -114,6 +146,17 @@ struct fuse_dlm_cache { * thread; the revoke paths only mark them killed. */ struct list_head pending; + /* Protects @pins and @fences */ + spinlock_t pin_lock; + /* + * Writers between confirming a grant and dirtying under it, each + * over the range it is about to write. See the pin comment above. + */ + struct list_head pins; + /* Revokes in progress, each over the range it takes away */ + struct list_head fences; + /* Both directions: pins draining, and the fences they wait on */ + wait_queue_head_t pin_wq; }; /* Initialize a page cache lock manager */ @@ -150,6 +193,35 @@ void fuse_dlm_request_abort(struct fuse_inode *inode, int fuse_dlm_unlock_range(struct fuse_inode *inode, uint64_t start, uint64_t end); +/* + * Hold the grants over [@offset, @offset + @length) against revocation + * until fuse_dlm_unpin(), which drops the pin this task last took. @pin + * is caller-owned storage, live until then. fuse_dlm_pin() waits out a + * revoke overlapping that range and must not be called with a folio + * held; fuse_dlm_trypin() never sleeps and fails instead. Neither may + * be held across a DLM request: that request is answered by the server + * the revoke came from. + */ +void fuse_dlm_pin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +bool fuse_dlm_trypin(struct fuse_inode *inode, struct fuse_dlm_span *pin, + loff_t offset, size_t length); +void fuse_dlm_unpin(struct fuse_inode *inode); + +/* + * Fence the writers that hold a grant over [@offset, @offset + @len) but + * have not dirtied under it yet, for the duration of a revoke. @len <= 0 + * means to EOF, as in fuse_notify_inval_inode(). @fence is caller-owned + * storage, live until the matching end. Between these the caller may + * drop coverage over that range knowing nothing will be dirtied under + * what it drops, and a write outside it is left alone. + */ +void fuse_dlm_revoke_begin(struct fuse_inode *inode, + struct fuse_dlm_span *fence, loff_t offset, + loff_t len); +void fuse_dlm_revoke_end(struct fuse_inode *inode, + struct fuse_dlm_span *fence); + /* Re-validate a fuse_get_dlm_lock() grant against the live lock tree */ bool fuse_dlm_lock_is_held(struct fuse_inode *inode, loff_t offset, size_t length, enum fuse_page_lock_mode mode); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index cef11f9cb170c7..0fae65e89fbe7d 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -46,11 +46,29 @@ */ extern const char fuse_notify_ctx_key[]; -static inline void *fuse_notify_ctx_enter(void) +/* + * What a task driving page cache work for a NOTIFY invalidate carries. + * + * @key tells it apart from anything else parked in journal_info. The range + * is the one being revoked: the lock over it is still this client's until + * the handler returns, so writeback of it need not ask for a grant, while + * anything outside it must. + */ +struct fuse_notify_ctx { + const char *key; + loff_t start; + loff_t end; /* inclusive; LLONG_MAX to EOF */ +}; + +static inline void *fuse_notify_ctx_enter(struct fuse_notify_ctx *ctx, + loff_t start, loff_t end) { void *old = current->journal_info; - current->journal_info = (void *)fuse_notify_ctx_key; + ctx->key = fuse_notify_ctx_key; + ctx->start = start; + ctx->end = end; + current->journal_info = ctx; return old; } @@ -59,9 +77,24 @@ static inline void fuse_notify_ctx_leave(void *old) current->journal_info = old; } +static inline struct fuse_notify_ctx *fuse_notify_ctx(void) +{ + struct fuse_notify_ctx *ctx = current->journal_info; + + return (ctx && ctx->key == fuse_notify_ctx_key) ? ctx : NULL; +} + static inline bool fuse_in_notify_ctx(void) { - return current->journal_info == (void *)fuse_notify_ctx_key; + return fuse_notify_ctx(); +} + +/* Is [@pos, @pos + @len) the range the revoke in progress is taking away? */ +static inline bool fuse_in_notify_range(loff_t pos, unsigned int len) +{ + struct fuse_notify_ctx *ctx = fuse_notify_ctx(); + + return ctx && pos >= ctx->start && pos + len - 1 <= ctx->end; } /** Default max number of pages that can be used in a single read request */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 391c2cf432c341..a8dc8742d22d6a 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -995,7 +995,10 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * here must not send a DLM request. See * fuse_iomap_writeback_range(). */ - void *notify_ctx = fuse_notify_ctx_enter(); + struct fuse_notify_ctx ctx; + struct fuse_dlm_span fence; + void *notify_ctx; + bool fenced; pg_start = offset >> PAGE_SHIFT; if (len <= 0) @@ -1006,6 +1009,37 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, /* Byte bounds of the same region */ end_byte = len <= 0 ? LLONG_MAX : offset + len - 1; + notify_ctx = fuse_notify_ctx_enter(&ctx, offset, end_byte); + + /* + * Fence the writers that hold a grant over this range but + * have not dirtied under it yet. Their bytes are in no page + * cache and on no wire, so nothing below can find them, and + * once the grant is gone they would go out behind the + * handover. Published before the flush so what it drains is + * flushed with everything else. + * + * Only the writers over this range: a write elsewhere in the + * file holds a grant this handler does not touch, and neither + * waits for the other. + * + * Entered after fuse_notify_ctx_enter(): the page cache work + * below is this handler's own and must not fence itself. + * + * A fenced writer may be waiting for a FUSE_READ or a + * FUSE_WRITE reply, so this waits on the server the same way + * the flush below does. + * + * Regular files only: the record shares the readdir cache + * union arm and exists nowhere else. Latched into a local, + * since fc->dlm can be cleared while this runs and the fence + * has to come off the list either way. + */ + fenced = S_ISREG(inode->i_mode) && fc->dlm && + fc->writeback_cache; + if (fenced) + fuse_dlm_revoke_begin(fi, &fence, offset, len); + /* * A data invalidation means another (remote) entity is * modifying the file. Two things happen here: @@ -1078,6 +1112,11 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, * waiting on. do_writepages() runs in this context, * so the grant is asked for before the revoke. * + * One pass is enough: the fence above has drained the + * writers that held a grant without having dirtied + * under it, and refuses new ones, so nothing can turn + * up dirty behind this. + * * Waited out here rather than left to the drop, which * launders when the record says the range may be dirty * and so waits for these same replies. One explicit @@ -1135,6 +1174,8 @@ int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, fuse_notify_invalidate_range(inode, pg_start, pg_end, true); } + if (fenced) + fuse_dlm_revoke_end(fi, &fence); fuse_notify_ctx_leave(notify_ctx); } iput(inode); From b901618ff0296e1395a29a98586f2b071d7e5d95 Mon Sep 17 00:00:00 2001 From: Horst Birthelmer Date: Tue, 1 Sep 2026 13:23:03 +0200 Subject: [PATCH 17/17] fuse: default large folios off A folio wider than a block carries per-block dirty state, and iomap_writeback_folio() clears all of it after ->writeback_range. A run fuse defers and puts back with folio_mark_dirty() comes back dirty over the whole folio, since iomap_dirty_folio() sets the whole range, so the next pass sends blocks this client never wrote: data read from the server, or zeros laid down above the size the file had at the time. Order-0 folios carry no per-block state, so a dirty folio is one this client wrote whole and the widening cannot happen. The parameter stays writable for anyone opting back in. Signed-off-by: Horst Birthelmer --- fs/fuse/inode.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index a8dc8742d22d6a..45b295efa4da4e 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -37,7 +37,14 @@ static bool __read_mostly enable_compound; module_param(enable_compound, bool, 0644); MODULE_PARM_DESC(enable_uring, "Enable fuse compounds"); -bool __read_mostly enable_large_folios = true; +/* + * A folio wider than a block carries per-block dirty state, which + * iomap_writeback_folio() clears whole after ->writeback_range. A run + * fuse defers and puts back with folio_mark_dirty() comes back dirty over + * the whole folio, so the next pass sends blocks this client never wrote. + * Off until there is a way to restore only the blocks that were dirty. + */ +bool __read_mostly enable_large_folios; module_param(enable_large_folios, bool, 0644); MODULE_PARM_DESC(enable_large_folios, "Enable large folios support");