From d319d537ee51e4e11dc052d97ec21c81321c4f3b Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Wed, 19 Aug 2026 08:06:09 +0000 Subject: [PATCH 1/2] fuse: add GPU Direct Storage (GDS) support Integrate with nvidia-fs to pass GPU buffer RDMA information to the filesystem server for direct GPU-to-storage transfers via NVIDIA GDS. GDS transfers data between GPU memory and storage through RDMA without CPU bounce buffers. This reduces host-memory use, latency, and CPU overhead for GPU workloads. - Negotiate GDS capability during filesystem initialization. - Obtain GPU RDMA information from nvidia-fs. - Pass that RDMA information to the server in read and write request extensions. --- fs/fuse/Makefile | 2 +- fs/fuse/file.c | 42 ++++++++++- fs/fuse/fuse_gds.c | 155 ++++++++++++++++++++++++++++++++++++++ fs/fuse/fuse_gds.h | 63 ++++++++++++++++ fs/fuse/fuse_i.h | 14 ++++ fs/fuse/inode.c | 5 +- include/uapi/linux/fuse.h | 6 ++ 7 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 fs/fuse/fuse_gds.c create mode 100644 fs/fuse/fuse_gds.h diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile index 64dad526c0b8ce..c0cf4cdadd4f91 100644 --- a/fs/fuse/Makefile +++ b/fs/fuse/Makefile @@ -10,7 +10,7 @@ obj-$(CONFIG_FUSE_FS) += fuse.o obj-$(CONFIG_CUSE) += cuse.o obj-$(CONFIG_VIRTIO_FS) += virtiofs.o -fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o +fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o fuse_gds.o fuse-y += iomode.o fuse-$(CONFIG_FUSE_DAX) += dax.o fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 55ab5e9cf61715..e28c0b77266678 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -8,6 +8,7 @@ #include "fuse_i.h" #include "fuse_dlm_cache.h" +#include "fuse_gds.h" #include #include @@ -669,6 +670,23 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end, return err; } +static void fuse_readwrite_args_fill_gds_ext(struct fuse_io_args *ia) +{ + struct fuse_args *args = &ia->ap.args; + struct fuse_ext_header *ext_header = (struct fuse_ext_header *)ia->readwrite_in_gds_ext; + + /* + * The RDMA information data is already filled by the fuse_gds_get_gpu_sglist_rdma_info + * function. Fill the extension header here. + */ + ext_header->type = FUSE_EXT_READ_WRITE_GDS; + ext_header->size = FUSE_EXT_READ_WRITE_GDS_SIZE; + + args->is_ext = 1; + args->ext_idx = args->in_numargs++; + args->in_args[args->ext_idx].size = ext_header->size; + args->in_args[args->ext_idx].value = ext_header; +} void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode) { @@ -687,6 +705,9 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->out_argvar = true; args->out_numargs = 1; args->out_args[0].size = count; + + if (ia->ap.args.use_gds) + fuse_readwrite_args_fill_gds_ext(ia); } static void fuse_release_user_pages(struct fuse_args_pages *ap, @@ -1126,16 +1147,25 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, ia->write.in.size = count; args->opcode = FUSE_WRITE; args->nodeid = ff->nodeid; - args->in_numargs = 2; + args->in_numargs = 0; if (ff->fm->fc->minor < 9) args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; else args->in_args[0].size = sizeof(ia->write.in); args->in_args[0].value = &ia->write.in; - args->in_args[1].size = count; + args->in_numargs++; + + if (ia->ap.args.use_gds) + fuse_readwrite_args_fill_gds_ext(ia); + + /* inpage argument must be the last one */ + args->in_args[args->in_numargs++].size = count; + args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; + + } static unsigned int fuse_write_flags(struct kiocb *iocb) @@ -1663,6 +1693,14 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, if (err && !nbytes) break; + if (fc->gds_support && fuse_is_gds_buffer(&ia->ap)) { + err = fuse_gds_get_gpu_sglist_rdma_info(fc, write, ia); + if (err) { + fuse_release_user_pages(&ia->ap, io->should_dirty); + break; + } + } + if (write) { if (!capable(CAP_FSETID)) ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; diff --git a/fs/fuse/fuse_gds.c b/fs/fuse/fuse_gds.c new file mode 100644 index 00000000000000..72c7b87679a82c --- /dev/null +++ b/fs/fuse/fuse_gds.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2026 DataDirect Networks. + */ + +#include +#include +#include "fuse_i.h" + +#define GDS_MOCK_TEST 0 + +/* NVIDIA GPU Direct Storage interface and operations */ + +static atomic_t nvfs_ops_refcnt = ATOMIC_INIT(0); +static struct nvfs_dma_rw_ops *nvfs_ops = NULL; + +static inline struct nvfs_dma_rw_ops* get_nvfs_dma_ops(void) +{ + struct nvfs_dma_rw_ops *ops; + + rcu_read_lock(); + ops = rcu_dereference(nvfs_ops); + if (ops) + atomic_inc(&nvfs_ops_refcnt); + rcu_read_unlock(); + return ops; +} + +static inline void put_nvfs_dma_ops(void) +{ + atomic_dec(&nvfs_ops_refcnt); +} + +int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops) +{ + if (!ops) + return -EINVAL; + + rcu_assign_pointer(nvfs_ops, ops); + return 0; +} + +void fuse_unregister_nvfs_dma_ops(void) +{ + rcu_assign_pointer(nvfs_ops, NULL); + synchronize_rcu(); + while (atomic_read(&nvfs_ops_refcnt) > 0) + msleep(100); +} +EXPORT_SYMBOL_GPL(fuse_register_nvfs_dma_ops); +EXPORT_SYMBOL_GPL(fuse_unregister_nvfs_dma_ops); + +/* GPU buffer detection and DMA scatter-gather mapping operations */ + +#if !GDS_MOCK_TEST +static bool nvfs_dma_ops_is_gds_page(struct page *page) +{ + struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops(); + if (ops) { + bool ret = ops->nvfs_is_gpu_page(page); + put_nvfs_dma_ops(); + return ret; + } + return false; +} + +bool fuse_is_gds_buffer(struct fuse_args_pages *ap) +{ + struct page **pages = ap->pages; + unsigned int num_pages = ap->num_pages; + + return ap->args.user_pages && num_pages > 0 && nvfs_dma_ops_is_gds_page(pages[0]); +} + +static int nvfs_get_gpu_sglist_rdma_info(struct scatterlist *sglist, + int nents, + struct nvfs_rdma_info *rdma_infop) +{ + struct nvfs_dma_rw_ops *ops = get_nvfs_dma_ops(); + int rc = -EIO; + if (ops) { + /* The function returns the number of processed entries upon successful completion */ + rc = ops->nvfs_get_gpu_sglist_rdma_info(sglist, nents, rdma_infop); + put_nvfs_dma_ops(); + } + return rc > 0 ? 0: -EIO; +} + +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia) +{ + struct sg_table sgt; + struct scatterlist *sgl; + int rc = 0; + unsigned int i; + + if (!ia->ap.num_pages) { + return -EINVAL; + } + + ia->ap.args.use_gds = 0; + if (sg_alloc_table(&sgt, ia->ap.num_pages, GFP_KERNEL)) { + rc = -ENOMEM; + goto out; + } + + sgl = sgt.sgl; + for(i = 0; i < ia->ap.num_pages; i++) { + sg_set_page(sgl, ia->ap.pages[i], ia->ap.descs[i].length, ia->ap.descs[i].offset); + sgl = sg_next(sgl); + } + + rc = nvfs_get_gpu_sglist_rdma_info(sgt.sgl, ia->ap.num_pages, fuse_get_readwrite_rdma_info(ia)); + if (rc) { + goto out_sgt; + } + + ia->ap.args.use_gds = 1; + +out_sgt: + sg_free_table(&sgt); +out: + return rc; +} + +#else +bool fuse_is_gds_buffer(struct fuse_args_pages *ap) +{ + return true; +} + +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia) +{ + struct nvfs_rdma_info *rdma_infop = fuse_get_readwrite_rdma_info(ia); + unsigned int total_len = 0; + for (unsigned int i = 0; i < ia->ap.num_pages; i++) { + total_len += ia->ap.descs[i].length; + } + rdma_infop->version = 2; + rdma_infop->flags = 1; + rdma_infop->lid = 0; + rdma_infop->qp_num = 3; + rdma_infop->rem_vaddr = 4; + rdma_infop->size = total_len; + rdma_infop->rkey = 5; + rdma_infop->gid[0] = 6; + rdma_infop->gid[1] = 7; + rdma_infop->dc_key = 8; + + ia->ap.args.use_gds = 1; + return 0; +} +#endif diff --git a/fs/fuse/fuse_gds.h b/fs/fuse/fuse_gds.h new file mode 100644 index 00000000000000..4af097483c225b --- /dev/null +++ b/fs/fuse/fuse_gds.h @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2026 DataDirect Networks. + */ + +#ifndef _FS_FUSE_GDS_H +#define _FS_FUSE_GDS_H + +#include + +struct fuse_args_pages; +struct fuse_conn; +struct fuse_io_args; +struct request; + +struct nvfs_rdma_info +{ + uint8_t version; /* to support future changes to structure */ + uint8_t flags; /* if bit 0 != 0, then gid field is valid */ + uint16_t lid; /* subnet local identifier of the client node port */ + uint32_t qp_num; /* QP number of DCT on the client node */ + uint64_t rem_vaddr; /* remote address */ + uint32_t size; + uint32_t rkey; + uint64_t gid[2]; /* 16-byte global identifier of the client node port */ + uint32_t dc_key; +}; + +struct nvfs_dma_rw_ops { + unsigned long long ft_bmap; /* feature bitmap */ + + int (*nvfs_blk_rq_map_sg) (struct request_queue *q, + struct request *req, + struct scatterlist *sglist); + + int (*nvfs_dma_map_sg_attrs) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir, + unsigned long attrs); + + int (*nvfs_dma_unmap_sg) (struct device *device, + struct scatterlist *sglist, + int nents, + enum dma_data_direction dma_dir); + + bool (*nvfs_is_gpu_page) (struct page *); + unsigned int (*nvfs_gpu_index) (struct page *page); + unsigned int (*nvfs_device_priority) (struct device *dev, unsigned int dev_index); + + int (*nvfs_get_gpu_sglist_rdma_info) (struct scatterlist *sglist, + int nents, + struct nvfs_rdma_info *rdma_infop); +}; + +bool fuse_is_gds_buffer(struct fuse_args_pages *ap); +int fuse_gds_get_gpu_sglist_rdma_info(struct fuse_conn *fc, bool write, + struct fuse_io_args *ia); +int fuse_register_nvfs_dma_ops(struct nvfs_dma_rw_ops *ops); +void fuse_unregister_nvfs_dma_ops(void); + +#endif diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c8c0b5a1477968..1556c8da21e1cf 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -32,6 +32,7 @@ #include #include #include "fuse_dlm_cache.h" +#include "fuse_gds.h" /** Default max number of pages that can be used in a single read request */ #define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 @@ -324,6 +325,7 @@ struct fuse_args { bool may_block:1; bool is_ext:1; bool is_pinned:1; + bool use_gds:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); @@ -879,6 +881,9 @@ struct fuse_conn { /* Is synchronous FUSE_INIT allowed? */ unsigned int sync_init:1; + /* Does the filesystem support GDS? */ + unsigned int gds_support:1; + /* Use io_uring for communication */ unsigned int io_uring; @@ -1115,6 +1120,10 @@ struct fuse_forget_link *fuse_alloc_forget(void); /* * Initialize READ or READDIR request */ + +#define FUSE_EXT_READ_WRITE_GDS_SIZE \ + FUSE_REC_ALIGN(sizeof(struct fuse_ext_header) + sizeof(struct nvfs_rdma_info)) + struct fuse_io_args { union { struct { @@ -1130,11 +1139,16 @@ struct fuse_io_args { struct fuse_args_pages ap; struct fuse_io_priv *io; struct fuse_file *ff; + uint8_t readwrite_in_gds_ext[FUSE_EXT_READ_WRITE_GDS_SIZE]; }; void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode); +static inline struct nvfs_rdma_info *fuse_get_readwrite_rdma_info(struct fuse_io_args *ia) +{ + return (struct nvfs_rdma_info *)(ia->readwrite_in_gds_ext + sizeof(struct fuse_ext_header)); +} /* * Helper functions to initialize fuse_args for common operations */ diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index 82c84a81b6e2d3..da81e0e99a8697 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1619,6 +1619,8 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, fc->inval_inode_entries = 1; if (flags & FUSE_EXPIRE_INODE_ENTRY) fc->expire_inode_entries = 1; + if (flags & FUSE_GDS_SUPPORT) + fc->gds_support = 1; } else { ra_pages = fc->max_read / PAGE_SIZE; fc->no_lock = 1; @@ -1669,7 +1671,8 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | FUSE_NO_EXPORT_SUPPORT | FUSE_INVAL_INODE_ENTRY | - FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q; + FUSE_EXPIRE_INODE_ENTRY | FUSE_URING_REDUCED_Q | + FUSE_GDS_SUPPORT; #ifdef CONFIG_FUSE_DAX if (fm->fc->dax) flags |= FUSE_MAP_ALIGNMENT; diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index d472df6370a400..b359b72844244b 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -482,6 +482,7 @@ struct fuse_file_lock { #define FUSE_URING_REDUCED_Q (1ULL << 59) #define FUSE_INVAL_INODE_ENTRY (1ULL << 60) #define FUSE_EXPIRE_INODE_ENTRY (1ULL << 61) +#define FUSE_GDS_SUPPORT (1ULL << 62) /** * CUSE INIT request/reply flags @@ -604,6 +605,7 @@ enum fuse_ext_type { /* Types 0..31 are reserved for fuse_secctx_header */ FUSE_MAX_NR_SECCTX = 31, FUSE_EXT_GROUPS = 32, + FUSE_EXT_READ_WRITE_GDS = 100, }; enum fuse_opcode { @@ -848,6 +850,10 @@ struct fuse_read_in { uint32_t padding; }; +struct fuse_gds_read_out { + uint64_t size; +}; + #define FUSE_COMPAT_WRITE_IN_SIZE 24 struct fuse_write_in { From 484a26f54840a1756165006d601e7f1a8805881c Mon Sep 17 00:00:00 2001 From: Cheng Ding Date: Mon, 31 Aug 2026 07:13:36 +0000 Subject: [PATCH 2/2] fuse: fix payload buffer overflow OOPS and GDS copy Extensions were copied via the payload buffer, sharing the same buffer as the page payload. This caused an OOPS when the combined size of extensions and write I/O exceeded fuse_max_write for fuse over uring. The old kernel code had no such issue because its extensions were used by FUSE requests without page payloads. Since fuse_uring_req_header is page-aligned, the solution is to add a dedicated 512-byte ext_in[] buffer for extensions (referred to as "new extension" in the code). Additionally, fix the GDS read/write code to skip page copy and handle GDS read replies correctly. --- fs/fuse/dev.c | 30 +++++++++++++++++++---- fs/fuse/dev_uring.c | 31 ++++++++++++++++++++++-- fs/fuse/file.c | 50 ++++++++++++++++++++++++++------------- fs/fuse/fuse_dev_i.h | 1 + fs/fuse/fuse_i.h | 4 ++++ include/uapi/linux/fuse.h | 8 ++++++- 6 files changed, 100 insertions(+), 24 deletions(-) diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 7fcd0bf4132bd4..e84bcdacd11d5a 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -498,6 +498,13 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) req->args = args; if (args->is_ext) req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; + if (args->has_new_ext) { + req->in.h.total_new_extlen = 0; + for (int i = 0; i < args->ext_numargs; i++) { + req->in.h.total_new_extlen += args->ext_in_args[i].size / 8; + } + BUG_ON(req->in.h.total_new_extlen * 8 > FUSE_URING_EXT_IN_SZ); + } if (args->end) __set_bit(FR_ASYNC, &req->flags); @@ -1010,6 +1017,7 @@ static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) /* Copy request arguments to/from userspace buffer */ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, unsigned argpages, struct fuse_arg *args, + unsigned ext_numargs, struct fuse_arg *ext_args, int zeroing) { int err = 0; @@ -1017,11 +1025,21 @@ int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, for (i = 0; !err && i < numargs; i++) { struct fuse_arg *arg = &args[i]; - if (i == numargs - 1 && argpages) - err = fuse_copy_pages(cs, arg->size, zeroing); - else + if (i < numargs - 1 || !argpages) err = fuse_copy_one(cs, arg->value, arg->size); } + + /* io_uring requests carry extensions in the request header, so ext_numargs is zero for them. */ + for (i = 0; !err && i < ext_numargs; i++) { + struct fuse_arg *arg = &ext_args[i]; + err = fuse_copy_one(cs, arg->value, arg->size); + } + + /* Copy the page-backed payload last. */ + if (numargs > 0 && !err && argpages) { + struct fuse_arg *arg = &args[numargs - 1]; + err = fuse_copy_pages(cs, arg->size, zeroing); + } return err; } @@ -1296,7 +1314,9 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); if (!err) err = fuse_copy_args(cs, args->in_numargs, args->in_pages, - (struct fuse_arg *) args->in_args, 0); + (struct fuse_arg *) args->in_args, + args->ext_numargs, + (struct fuse_arg *) args->ext_in_args, 0); fuse_copy_finish(cs); spin_lock(&fpq->lock); clear_bit(FR_LOCKED, &req->flags); @@ -1867,7 +1887,7 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, lastarg->size -= diffsize; } return fuse_copy_args(cs, args->out_numargs, args->out_pages, - args->out_args, args->page_zeroing); + args->out_args, 0, NULL, args->page_zeroing); } /* diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 6b2de8b3e4ec8e..f3379103d931a7 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -732,6 +732,8 @@ static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring, struct fuse_copy_state cs; struct fuse_args *args = req->args; struct fuse_in_arg *in_args = args->in_args; + struct fuse_in_arg *ext_in_args = args->ext_in_args; + char *ext_in = headers->ext_in; int num_args = args->in_numargs; int err; @@ -757,9 +759,17 @@ static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring, num_args--; } + if (args->has_new_ext) { + for (int i = 0; i < args->ext_numargs; i++) { + memcpy(ext_in, ext_in_args->value, ext_in_args->size); + ext_in += ext_in_args->size; + ext_in_args++; + } + } + /* copy the payload */ err = fuse_copy_args(&cs, num_args, args->in_pages, - (struct fuse_arg *)in_args, 0); + (struct fuse_arg *)in_args, 0, NULL, 0); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); goto copy_finish; @@ -782,6 +792,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_copy_state cs; struct fuse_args *args = req->args; struct fuse_in_arg *in_args = args->in_args; + struct fuse_in_arg *ext_in_args = args->ext_in_args; + char *ext_in = ent->headers->ext_in; int num_args = args->in_numargs; int err; struct iov_iter iter; @@ -820,9 +832,24 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, num_args--; } + /* copy the new extension arguments */ + if (args->has_new_ext) { + for (int i = 0; i < args->ext_numargs; i++) { + err = copy_to_user(ext_in, ext_in_args->value, + ext_in_args->size); + if (err) { + pr_info_ratelimited( + "Copying the extension header failed.\n"); + return -EFAULT; + } + ext_in += ext_in_args->size; + ext_in_args++; + } + } + /* copy the payload */ err = fuse_copy_args(&cs, num_args, args->in_pages, - (struct fuse_arg *)in_args, 0); + (struct fuse_arg *)in_args, 0, NULL, 0); if (err) { pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); goto copy_finish; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e28c0b77266678..6f4cdbd6cb8838 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -682,10 +682,13 @@ static void fuse_readwrite_args_fill_gds_ext(struct fuse_io_args *ia) ext_header->type = FUSE_EXT_READ_WRITE_GDS; ext_header->size = FUSE_EXT_READ_WRITE_GDS_SIZE; - args->is_ext = 1; - args->ext_idx = args->in_numargs++; - args->in_args[args->ext_idx].size = ext_header->size; - args->in_args[args->ext_idx].value = ext_header; + args->has_new_ext = 1; + /* The data buffer is already in the GPU memory, no need to copy it */ + args->in_pages = false; + args->out_pages = false; + args->ext_in_args[args->ext_numargs].size = ext_header->size; + args->ext_in_args[args->ext_numargs].value = ext_header; + args->ext_numargs++; } void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, size_t count, int opcode) @@ -704,10 +707,15 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, args->in_args[0].value = &ia->read.in; args->out_argvar = true; args->out_numargs = 1; - args->out_args[0].size = count; - - if (ia->ap.args.use_gds) + if (ia->ap.args.use_gds) { + /* File data is transferred through RDMA; out_args[0] returns the read size. */ fuse_readwrite_args_fill_gds_ext(ia); + args->out_args[0].size = sizeof(ia->read.out); + args->out_args[0].value = &ia->read.out; + } + else { + args->out_args[0].size = count; + } } static void fuse_release_user_pages(struct fuse_args_pages *ap, @@ -894,6 +902,7 @@ static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, struct file *file = ia->io->iocb->ki_filp; struct fuse_file *ff = file->private_data; struct fuse_mount *fm = ff->fm; + ssize_t ret; fuse_read_args_fill(ia, file, pos, count, FUSE_READ); if (owner != NULL) { @@ -904,7 +913,16 @@ static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, if (ia->io->async) return fuse_async_req_send(fm, ia, count); - return fuse_simple_request(fm, &ia->ap.args); + ret = fuse_simple_request(fm, &ia->ap.args); + + /* + * GDS reads return the fixed-size reply payload; regular reads return + * their byte count. Normalize a successful GDS reply to its byte count. + */ + if (ia->ap.args.use_gds && ret == sizeof(ia->read.out)) { + ret = (ssize_t) ia->read.out.size; + } + return ret; } static void fuse_read_update_size(struct inode *inode, loff_t size, @@ -1147,20 +1165,20 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, ia->write.in.size = count; args->opcode = FUSE_WRITE; args->nodeid = ff->nodeid; - args->in_numargs = 0; if (ff->fm->fc->minor < 9) args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; else args->in_args[0].size = sizeof(ia->write.in); args->in_args[0].value = &ia->write.in; - args->in_numargs++; - - if (ia->ap.args.use_gds) + if (ia->ap.args.use_gds) { + /* No need to copy the data buffer */ + args->in_numargs = 1; fuse_readwrite_args_fill_gds_ext(ia); - - /* inpage argument must be the last one */ - args->in_args[args->in_numargs++].size = count; - + } + else { + args->in_numargs = 2; + args->in_args[1].size = count; + } args->out_numargs = 1; args->out_args[0].size = sizeof(ia->write.out); args->out_args[0].value = &ia->write.out; diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h index 316553ed074722..e8c14eda8bdc4f 100644 --- a/fs/fuse/fuse_dev_i.h +++ b/fs/fuse/fuse_dev_i.h @@ -63,6 +63,7 @@ void fuse_copy_init(struct fuse_copy_state *cs, int write, void fuse_copy_finish(struct fuse_copy_state *cs); int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs, unsigned int argpages, struct fuse_arg *args, + unsigned int ext_numargs, struct fuse_arg *ext_args, int zeroing); int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, unsigned int nbytes); diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index 1556c8da21e1cf..8baabfb75d86d5 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -313,6 +313,7 @@ struct fuse_args { uint8_t in_numargs; uint8_t out_numargs; uint8_t ext_idx; + uint8_t ext_numargs; /* number of the new extensions */ bool force:1; bool noreply:1; bool nocreds:1; @@ -326,8 +327,10 @@ struct fuse_args { bool is_ext:1; bool is_pinned:1; bool use_gds:1; + bool has_new_ext:1; struct fuse_in_arg in_args[4]; struct fuse_arg out_args[2]; + struct fuse_in_arg ext_in_args[4]; /* new extensions */ void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); }; @@ -1128,6 +1131,7 @@ struct fuse_io_args { union { struct { struct fuse_read_in in; + struct fuse_gds_read_out out; u64 attr_ver; } read; struct { diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h index b359b72844244b..a3c09a6675e93d 100644 --- a/include/uapi/linux/fuse.h +++ b/include/uapi/linux/fuse.h @@ -1041,7 +1041,9 @@ struct fuse_in_header { uint32_t gid; uint32_t pid; uint16_t total_extlen; /* length of extensions in 8byte units */ - uint16_t padding; + uint8_t total_new_extlen; /* length of new extensions in 8byte units + (fuse_uring_req_header::ext_in) */ + uint8_t padding; }; struct fuse_out_header { @@ -1299,6 +1301,7 @@ struct fuse_compound_out { */ #define FUSE_URING_IN_OUT_HEADER_SZ 128 #define FUSE_URING_OP_IN_OUT_SZ 128 +#define FUSE_URING_EXT_IN_SZ 512 /* Used as part of the fuse_uring_req_header */ struct fuse_uring_ent_in_out { @@ -1328,6 +1331,9 @@ struct fuse_uring_req_header { char op_in[FUSE_URING_OP_IN_OUT_SZ]; struct fuse_uring_ent_in_out ring_ent_in_out; + + /* extention space */ + char ext_in[FUSE_URING_EXT_IN_SZ]; }; /**