fuse: add GPU Direct Storage (GDS) support

fs/fuse/Makefile

@@ -10,7 +10,7 @@ obj-$(CONFIG_FUSE_FS) += fuse.o
 obj-$(CONFIG_CUSE) += cuse.o
 obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
-fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o
+fuse-y := dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o fuse_dlm_cache.o compound.o fuse_gds.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o

fs/fuse/dev.c

@@ -498,6 +498,13 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
 	req->args = args;
 	if (args->is_ext)
 		req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8;
+	if (args->has_new_ext) {
+		req->in.h.total_new_extlen = 0;
+		for (int i = 0; i < args->ext_numargs; i++) {
+			req->in.h.total_new_extlen += args->ext_in_args[i].size / 8;
+		}
+		BUG_ON(req->in.h.total_new_extlen * 8 > FUSE_URING_EXT_IN_SZ);
+	}
 	if (args->end)
 		__set_bit(FR_ASYNC, &req->flags);
 
@@ -1010,18 +1017,29 @@ static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
 /* Copy request arguments to/from userspace buffer */
 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
 		   unsigned argpages, struct fuse_arg *args,
+		   unsigned ext_numargs, struct fuse_arg *ext_args,
 		   int zeroing)
 {
 	int err = 0;
 	unsigned i;
 
 	for (i = 0; !err && i < numargs; i++)  {
 		struct fuse_arg *arg = &args[i];
-		if (i == numargs - 1 && argpages)
-			err = fuse_copy_pages(cs, arg->size, zeroing);
-		else
+		if (i < numargs - 1 || !argpages)
 			err = fuse_copy_one(cs, arg->value, arg->size);
 	}
+
+	/* io_uring requests carry extensions in the request header, so ext_numargs is zero for them. */
+	for (i = 0; !err && i < ext_numargs; i++)  {
+		struct fuse_arg *arg = &ext_args[i];
+		err = fuse_copy_one(cs, arg->value, arg->size);
+	}
+
+	/* Copy the page-backed payload last. */
+	if (numargs > 0 && !err && argpages) {
+	        struct fuse_arg *arg = &args[numargs - 1];
+	        err = fuse_copy_pages(cs, arg->size, zeroing);
+	}
 	return err;
 }
 
@@ -1296,7 +1314,9 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 	err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
 	if (!err)
 		err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
-				     (struct fuse_arg *) args->in_args, 0);
+				     (struct fuse_arg *) args->in_args,
+				     args->ext_numargs,
+				     (struct fuse_arg *) args->ext_in_args, 0);
 	fuse_copy_finish(cs);
 	spin_lock(&fpq->lock);
 	clear_bit(FR_LOCKED, &req->flags);
@@ -1867,7 +1887,7 @@ int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
 		lastarg->size -= diffsize;
 	}
 	return fuse_copy_args(cs, args->out_numargs, args->out_pages,
-			      args->out_args, args->page_zeroing);
+			      args->out_args, 0, NULL, args->page_zeroing);
 }
 
 /*

fs/fuse/dev_uring.c

@@ -732,6 +732,8 @@ static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring,
 	struct fuse_copy_state cs;
 	struct fuse_args *args = req->args;
 	struct fuse_in_arg *in_args = args->in_args;
+	struct fuse_in_arg *ext_in_args = args->ext_in_args;
+	char *ext_in = headers->ext_in;
 	int num_args = args->in_numargs;
 	int err;
 
@@ -757,9 +759,17 @@ static int fuse_uring_args_to_ring_pages(struct fuse_ring *ring,
 		num_args--;
 	}
 
+	if (args->has_new_ext) {
+		for (int i = 0; i < args->ext_numargs; i++) {
+			memcpy(ext_in, ext_in_args->value, ext_in_args->size);
+			ext_in += ext_in_args->size;
+			ext_in_args++;
+		}
+	}
+
 	/* copy the payload */
 	err = fuse_copy_args(&cs, num_args, args->in_pages,
-			     (struct fuse_arg *)in_args, 0);
+			     (struct fuse_arg *)in_args, 0, NULL, 0);
 	if (err) {
 		pr_info_ratelimited("%s fuse_copy_args failed\n", __func__);
 		goto copy_finish;
@@ -782,6 +792,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 	struct fuse_copy_state cs;
 	struct fuse_args *args = req->args;
 	struct fuse_in_arg *in_args = args->in_args;
+	struct fuse_in_arg *ext_in_args = args->ext_in_args;
+	char *ext_in = ent->headers->ext_in;
 	int num_args = args->in_numargs;
 	int err;
 	struct iov_iter iter;
@@ -820,9 +832,24 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 		num_args--;
 	}
 
+	/* copy the new extension arguments */
+	if (args->has_new_ext) {
+		for (int i = 0; i < args->ext_numargs; i++) {
+			err = copy_to_user(ext_in, ext_in_args->value,
+					   ext_in_args->size);
+			if (err) {
+				pr_info_ratelimited(
+					"Copying the extension header failed.\n");
+				return -EFAULT;
+			}
+			ext_in += ext_in_args->size;
+			ext_in_args++;
+		}
+	}
+
 	/* copy the payload */
 	err = fuse_copy_args(&cs, num_args, args->in_pages,
-			     (struct fuse_arg *)in_args, 0);
+			     (struct fuse_arg *)in_args, 0, NULL, 0);
 	if (err) {
 		pr_info_ratelimited("%s fuse_copy_args failed\n", __func__);
 		goto copy_finish;

fs/fuse/file.c

@@ -8,6 +8,7 @@
 
 #include "fuse_i.h"
 #include "fuse_dlm_cache.h"
+#include "fuse_gds.h"
 
 #include <linux/pagemap.h>
 #include <linux/slab.h>
@@ -669,6 +670,26 @@ static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 	return err;
 }
 
+static void fuse_readwrite_args_fill_gds_ext(struct fuse_io_args *ia)
+{
+	struct fuse_args *args = &ia->ap.args;
+	struct fuse_ext_header *ext_header = (struct fuse_ext_header *)ia->readwrite_in_gds_ext;
+
+	/*
+	 * The RDMA information data is already filled by the fuse_gds_get_gpu_sglist_rdma_info
+	 * function. Fill the extension header here.
+	 */
+	ext_header->type = FUSE_EXT_READ_WRITE_GDS;
+	ext_header->size = FUSE_EXT_READ_WRITE_GDS_SIZE;
+
+	args->has_new_ext = 1;
+	/* The data buffer is already in the GPU memory, no need to copy it */
+	args->in_pages = false;
+	args->out_pages = false;
+	args->ext_in_args[args->ext_numargs].size = ext_header->size;
+	args->ext_in_args[args->ext_numargs].value = ext_header;
+	args->ext_numargs++;
+}
 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 			 size_t count, int opcode)
 {
@@ -686,7 +707,15 @@ void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 	args->in_args[0].value = &ia->read.in;
 	args->out_argvar = true;
 	args->out_numargs = 1;
-	args->out_args[0].size = count;
+	if (ia->ap.args.use_gds) {
+		/* File data is transferred through RDMA; out_args[0] returns the read size. */
+		fuse_readwrite_args_fill_gds_ext(ia);
+		args->out_args[0].size = sizeof(ia->read.out);
+		args->out_args[0].value = &ia->read.out;
+	}
+	else {
+		args->out_args[0].size = count;
+	}
 }
 
 static void fuse_release_user_pages(struct fuse_args_pages *ap,
@@ -873,6 +902,7 @@ static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 	struct file *file = ia->io->iocb->ki_filp;
 	struct fuse_file *ff = file->private_data;
 	struct fuse_mount *fm = ff->fm;
+	ssize_t ret;
 
 	fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 	if (owner != NULL) {
@@ -883,7 +913,16 @@ static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 	if (ia->io->async)
 		return fuse_async_req_send(fm, ia, count);
 
-	return fuse_simple_request(fm, &ia->ap.args);
+	ret = fuse_simple_request(fm, &ia->ap.args);
+
+	/*
+	 * GDS reads return the fixed-size reply payload; regular reads return
+	 * their byte count. Normalize a successful GDS reply to its byte count.
+	 */
+	if (ia->ap.args.use_gds && ret == sizeof(ia->read.out)) {
+		ret = (ssize_t) ia->read.out.size;
+	}
+	return ret;
 }
 
 static void fuse_read_update_size(struct inode *inode, loff_t size,
@@ -1126,16 +1165,25 @@ static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
 	ia->write.in.size = count;
 	args->opcode = FUSE_WRITE;
 	args->nodeid = ff->nodeid;
-	args->in_numargs = 2;
 	if (ff->fm->fc->minor < 9)
 		args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
 	else
 		args->in_args[0].size = sizeof(ia->write.in);
 	args->in_args[0].value = &ia->write.in;
-	args->in_args[1].size = count;
+	if (ia->ap.args.use_gds) {
+		/* No need to copy the data buffer */
+		args->in_numargs = 1;
+		fuse_readwrite_args_fill_gds_ext(ia);
+	}
+	else {
+		args->in_numargs = 2;
+		args->in_args[1].size = count;
+	}
 	args->out_numargs = 1;
 	args->out_args[0].size = sizeof(ia->write.out);
 	args->out_args[0].value = &ia->write.out;
+
+
 }
 
 static unsigned int fuse_write_flags(struct kiocb *iocb)
@@ -1663,6 +1711,14 @@ ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
 		if (err && !nbytes)
 			break;
 
+		if (fc->gds_support && fuse_is_gds_buffer(&ia->ap)) {
+			err = fuse_gds_get_gpu_sglist_rdma_info(fc, write, ia);
+			if (err) {
+				fuse_release_user_pages(&ia->ap, io->should_dirty);
+				break;
+			}
+		}
+
 		if (write) {
 			if (!capable(CAP_FSETID))
 				ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;

fs/fuse/fuse_dev_i.h

@@ -63,6 +63,7 @@ void fuse_copy_init(struct fuse_copy_state *cs, int write,
 void fuse_copy_finish(struct fuse_copy_state *cs);
 int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs,
 		   unsigned int argpages, struct fuse_arg *args,
+		   unsigned int ext_numargs, struct fuse_arg *ext_args,
 		   int zeroing);
 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
 		       unsigned int nbytes);