fuse: make fuse max-write, max-read params tunable

Since kernel 4.20, the FUSE driver introduced the max-pages feature.
To enhance write performance, this patch exports two mount options
fuse-max-write and fuse-max-read to let users specify the maximum
write/read bytes for FUSE connection.

By ajudging fuse-max-writef/fuse-max-read, the max-pages parameter
for fuse connection will be changed accordingly. It helps to avoid
IO splits when IO size is larger than 128K.

Updates: #4500
Signed-off-by: chenjinhao <[email protected]>

Author: chen1195585098

contrib/fuse-include/fuse_kernel.h

@@ -285,6 +285,8 @@ struct fuse_file_lock {
 #define FUSE_MAX_PAGES		(1 << 22)
 #define FUSE_CACHE_SYMLINKS	(1 << 23)
 
+#define FUSE_MAX_MAX_PAGES 256
+
 /**
  * CUSE INIT request/reply flags
  *

glusterfsd/src/glusterfsd.c

@@ -271,6 +271,10 @@ static struct argp_option gf_options[] = {
     {"fuse-handle-copy_file_range", ARGP_FUSE_HANDLE_COPY_FILE_RANGE, "BOOL",
      OPTION_ARG_OPTIONAL | OPTION_HIDDEN,
      "enable the handler of the FUSE_COPY_FILE_RANGE message"},
+    {"fuse-max-write", ARGP_FUSE_MAX_WRITE_KEY, "INTEGER", OPTION_ARG_OPTIONAL,
+     "set fuse max-write bytes"},
+    {"fuse-max-read", ARGP_FUSE_MAX_READ_KEY, "INTEGER", OPTION_ARG_OPTIONAL,
+     "set fuse max-read bytes"},
     {0, 0, 0, 0, "Miscellaneous Options:"},
     {
         0,
@@ -566,6 +570,16 @@ set_fuse_mount_options(glusterfs_ctx_t *ctx, dict_t *options)
             break;
     }
 
+    if (cmd_args->fuse_max_write) {
+        DICT_SET_VAL(dict_set_uint32, options, "fuse-max-write",
+                     cmd_args->fuse_max_write, glusterfsd_msg_3);
+    }
+
+    if (cmd_args->fuse_max_read) {
+        DICT_SET_VAL(dict_set_uint32, options, "fuse-max-read",
+                     cmd_args->fuse_max_read, glusterfsd_msg_3);
+    }
+
     if (cmd_args->fs_display_name) {
         DICT_SET_VAL(dict_set_dynstr, options, "fs-display-name",
                      cmd_args->fs_display_name, glusterfsd_msg_3);
@@ -1463,6 +1477,33 @@ parse_opts(int key, char *arg, struct argp_state *state)
                          "unknown fuse handle copy_file_range setting \"%s\"",
                          arg);
             break;
+        case ARGP_FUSE_MAX_WRITE_KEY:
+            if (gf_string2uint32(arg, &cmd_args->fuse_max_write)) {
+                argp_failure(state, -1, 0, "unknown fuse max-write option %s",
+                             arg);
+            } else if ((cmd_args->fuse_max_write < 4096) ||
+                       (cmd_args->fuse_max_write > 1048576)) {
+                argp_failure(state, -1, 0,
+                             "Invalid fuse max-write bytes %s. "
+                             "Valid range: [\"4096, 1048576\"]",
+                             arg);
+            }
+
+            break;
+        case ARGP_FUSE_MAX_READ_KEY:
+            if (gf_string2uint32(arg, &cmd_args->fuse_max_read)) {
+                argp_failure(state, -1, 0, "unknown fuse max-read option %s",
+                             arg);
+            } else if ((cmd_args->fuse_max_read < 4096) ||
+                       (cmd_args->fuse_max_read > 1048576)) {
+                argp_failure(state, -1, 0,
+                             "Invalid fuse max-read bytes %s. "
+                             "Valid range: [\"4096, 1048576\"]",
+                             arg);
+            }
+
+            break;
+
     }
     return 0;
 }

glusterfsd/src/glusterfsd.h

@@ -118,6 +118,8 @@ enum argp_option_keys {
     ARGP_FUSE_INODE_TABLESIZE_KEY = 198,
     ARGP_FUSE_SETLK_HANDLE_INTERRUPT_KEY = 199,
     ARGP_FUSE_HANDLE_COPY_FILE_RANGE = 200,
+    ARGP_FUSE_MAX_WRITE_KEY = 201,
+    ARGP_FUSE_MAX_READ_KEY = 202,
 };
 
 int

libglusterfs/src/glusterfs/glusterfs.h

@@ -592,6 +592,8 @@ struct _cmd_args {
     int fuse_auto_inval;
 
     uint32_t fuse_dev_eperm_ratelimit_ns;
+    uint32_t fuse_max_write;
+    uint32_t fuse_max_read;
 
     bool global_threading;
     bool brick_mux;

xlators/mount/fuse/src/fuse-bridge.c

@@ -5154,6 +5154,15 @@ fuse_init(xlator_t *this, fuse_in_header_t *finh, void *msg,
     fino.max_readahead = 1 << 17;
     fino.max_write = 1 << 17;
     fino.flags = FUSE_ASYNC_READ | FUSE_POSIX_LOCKS;
+#if FUSE_KERNEL_MINOR_VERSION >= 28
+    if (fini->minor >= 28) {
+        fino.max_readahead = priv->fuse_max_read;
+        fino.max_write = priv->fuse_max_write;
+        fino.max_pages = max(priv->fuse_max_write, priv->fuse_max_read) / sysconf(_SC_PAGESIZE);
+        if (fino.max_pages == FUSE_MAX_MAX_PAGES)
+            fino.flags |= FUSE_MAX_PAGES;
+    }
+#endif
 #if FUSE_KERNEL_MINOR_VERSION >= 17
     if (fini->minor >= 17)
         fino.flags |= FUSE_FLOCK_LOCKS;
@@ -6090,14 +6099,12 @@ fuse_thread_proc(void *data)
     struct pollfd pfd[2] = {{
         0,
     }};
-    uint32_t psize;
 
     this = data;
     priv = this->private;
 
     THIS = this;
 
-    psize = ((struct iobuf_pool *)this->ctx->iobuf_pool)->default_page_size;
     priv->msg0_len_p = &msg0_size;
 
     for (;;) {
@@ -6153,7 +6160,8 @@ fuse_thread_proc(void *data)
            size from 'fuse', which is as of today 128KB. If we bring in
            support for higher block sizes support, then we should be
            changing this one too */
-        iobuf = iobuf_get(this->ctx->iobuf_pool);
+        iobuf = iobuf_get2(this->ctx->iobuf_pool,
+                           max(priv->fuse_max_write, priv->fuse_max_read));
 
         /* Add extra 512 byte to the first iov so that it can
          * accommodate "ordinary" non-write requests. It's not
@@ -6177,7 +6185,7 @@ fuse_thread_proc(void *data)
         iov_in[1].iov_base = iobuf->ptr;
 
         iov_in[0].iov_len = msg0_size;
-        iov_in[1].iov_len = psize;
+        iov_in[1].iov_len = max(priv->fuse_max_write, priv->fuse_max_read);
 
         res = sys_readv(priv->fd, iov_in, 2);
 
@@ -6910,6 +6918,12 @@ init(xlator_t *this_xl)
     GF_OPTION_INIT("fuse-dev-eperm-ratelimit-ns",
                    priv->fuse_dev_eperm_ratelimit_ns, uint32, cleanup_exit);
 
+    GF_OPTION_INIT("fuse-max-write", priv->fuse_max_write, uint32,
+                   cleanup_exit);
+
+    GF_OPTION_INIT("fuse-max-read", priv->fuse_max_read, uint32,
+                   cleanup_exit);
+
     /* user has set only background-qlen, not congestion-threshold,
        use the fuse kernel driver formula to set congestion. ie, 75% */
     if (dict_get(this_xl->options, "background-qlen") &&
@@ -6954,11 +6968,11 @@ init(xlator_t *this_xl)
         goto cleanup_exit;
     }
 
-    gf_asprintf(&mnt_args, "%s%s%s%sallow_other,max_read=131072",
+    gf_asprintf(&mnt_args, "%s%s%s%sallow_other,max_read=%lu",
                 priv->acl ? "" : "default_permissions,",
                 priv->read_only ? "ro," : "",
                 priv->fuse_mountopts ? priv->fuse_mountopts : "",
-                priv->fuse_mountopts ? "," : "");
+                priv->fuse_mountopts ? "," : "", priv->fuse_max_read);
     if (!mnt_args)
         goto cleanup_exit;
 
@@ -7198,6 +7212,22 @@ struct volume_options options[] = {
         .max = 64,
         .description = "Sets fuse reader thread count.",
     },
+    {
+        .key = {"fuse-max-write"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "131072",
+        .min = 4096,
+        .max = 1048576,
+        .description = "Sets fuse max-write bytes.",
+    },
+    {
+        .key = {"fuse-max-read"},
+        .type = GF_OPTION_TYPE_INT,
+        .default_value = "131072",
+        .min = 4096,
+        .max = 1048576,
+        .description = "Sets fuse max-read bytes.",
+    },
     {
         .key = {"kernel-writeback-cache"},
         .type = GF_OPTION_TYPE_BOOL,

xlators/mount/fuse/src/fuse-bridge.h

@@ -176,6 +176,8 @@ struct fuse_private {
     uint32_t inode_table_size;
     uint32_t invalidate_limit;
     uint32_t fuse_dev_eperm_ratelimit_ns;
+    uint32_t fuse_max_write;
+    uint32_t fuse_max_read;
 
     /* counters for fusdev errnos */
     uint8_t fusedev_errno_cnt[FUSEDEV_EMAXPLUS];

xlators/mount/fuse/utils/mount.glusterfs.in

@@ -332,6 +332,14 @@ start_glusterfs ()
         cmd_line=$(echo "$cmd_line --fuse-handle-copy_file_range=$fuse_handle_copy_file_range");
     fi
 
+    if [ -n "$fuse_max_write" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-max-write=$fuse_max_write");
+    fi
+
+    if [ -n "$fuse_max_read" ]; then
+        cmd_line=$(echo "$cmd_line --fuse-max-read=$fuse_max_read");
+    fi
+
     if [ -n "$process_name" ]; then
         cmd_line=$(echo "$cmd_line --process-name fuse.$process_name");
     else
@@ -641,6 +649,12 @@ with_options()
         "fuse-dev-eperm-ratelimit-ns")
             fuse_dev_eperm_ratelimit_ns=$value
             ;;
+        "fuse-max-write")
+            fuse_max_write=$value
+            ;;
+        "fuse-max-read")
+            fuse_max_read=$value
+            ;;
         "context"|"fscontext"|"defcontext"|"rootcontext")
             # standard SElinux mount options to pass to the kernel
             [ -z "$fuse_mountopts" ] || fuse_mountopts="$fuse_mountopts,"