From e0109fb7c095df9edaa2ce5d3ae3fa32e8f292ab Mon Sep 17 00:00:00 2001 From: Kislay Kishore Date: Thu, 10 Oct 2024 14:16:51 +0530 Subject: [PATCH] Clean up references to old config --- cmd/config_validation_test.go | 2 +- cmd/datatypes_parsing_test.go | 6 +- cmd/flags.go | 677 ------------------ cmd/flags_test.go | 329 --------- cmd/legacy_main.go | 61 -- cmd/legacy_param_mapper.go | 188 ----- cmd/legacy_param_mapper_test.go | 658 ----------------- cmd/root.go | 8 +- cmd/root_test.go | 34 +- go.mod | 3 - go.sum | 6 - internal/config/mount_config.go | 211 ------ internal/config/testdata/empty_file.yaml | 0 .../invalid_disable_parallel_dirops.yaml | 2 - .../invalid_ignore_interrupts.yaml | 2 - .../unset_disable_parallel_dirops.yaml | 2 - .../unset_ignore_interrupts.yaml | 2 - .../unset_kernel_list_cache_ttl.yaml | 1 - .../valid_kernel_list_cache_ttl.yaml | 2 - .../gcs_auth/invalid_anonymous_access.yaml | 2 - .../gcs_auth/unset_anonymous_access.yaml | 2 - .../invalid_conn_pool_size.yaml | 3 - .../gcs_connection/unset_conn_pool_size.yaml | 3 - internal/config/testdata/invalid_config.yaml | 1 - .../config/testdata/invalid_log_config.yaml | 4 - .../invalid_unexpectedfield_config.yaml | 5 - ...e_config_stat-cache-max-size-mb_unset.yaml | 2 - .../metadata_cache_config_ttl-unset.yaml | 1 - ...e_config_type-cache-max-size-mb_unset.yaml | 2 - .../metrics_config/unset_prometheus_port.yaml | 2 - internal/config/testdata/valid_config.yaml | 39 - internal/config/yaml_parser.go | 95 --- internal/config/yaml_parser_test.go | 254 ------- internal/fs/inode/dir_test.go | 9 +- internal/fs/inode/hns_dir_test.go | 5 +- main.go | 25 +- main_test.go | 150 ---- tools/build_gcsfuse/main_test.go | 33 +- .../mounting/gcsfuse_test.go | 79 -- 39 files changed, 42 insertions(+), 2868 deletions(-) delete mode 100644 cmd/flags.go delete mode 100644 cmd/flags_test.go delete mode 100644 cmd/legacy_param_mapper.go delete mode 100644 cmd/legacy_param_mapper_test.go delete mode 100644 internal/config/mount_config.go delete mode 100644 internal/config/testdata/empty_file.yaml delete mode 100644 internal/config/testdata/file_system_config/invalid_disable_parallel_dirops.yaml delete mode 100644 internal/config/testdata/file_system_config/invalid_ignore_interrupts.yaml delete mode 100644 internal/config/testdata/file_system_config/unset_disable_parallel_dirops.yaml delete mode 100644 internal/config/testdata/file_system_config/unset_ignore_interrupts.yaml delete mode 100644 internal/config/testdata/file_system_config/unset_kernel_list_cache_ttl.yaml delete mode 100644 internal/config/testdata/file_system_config/valid_kernel_list_cache_ttl.yaml delete mode 100644 internal/config/testdata/gcs_auth/invalid_anonymous_access.yaml delete mode 100644 internal/config/testdata/gcs_auth/unset_anonymous_access.yaml delete mode 100644 internal/config/testdata/gcs_connection/invalid_conn_pool_size.yaml delete mode 100644 internal/config/testdata/gcs_connection/unset_conn_pool_size.yaml delete mode 100644 internal/config/testdata/invalid_config.yaml delete mode 100644 internal/config/testdata/invalid_log_config.yaml delete mode 100644 internal/config/testdata/invalid_unexpectedfield_config.yaml delete mode 100644 internal/config/testdata/metadata_cache_config_stat-cache-max-size-mb_unset.yaml delete mode 100644 internal/config/testdata/metadata_cache_config_ttl-unset.yaml delete mode 100644 internal/config/testdata/metadata_cache_config_type-cache-max-size-mb_unset.yaml delete mode 100644 internal/config/testdata/metrics_config/unset_prometheus_port.yaml delete mode 100644 internal/config/testdata/valid_config.yaml delete mode 100644 internal/config/yaml_parser.go delete mode 100644 internal/config/yaml_parser_test.go delete mode 100644 main_test.go diff --git a/cmd/config_validation_test.go b/cmd/config_validation_test.go index 88d2a5f081..8e218031b5 100644 --- a/cmd/config_validation_test.go +++ b/cmd/config_validation_test.go @@ -38,7 +38,7 @@ func getConfigObject(t *testing.T, args []string) (*cfg.Config, error) { require.Nil(t, err) cmdArgs := append([]string{"gcsfuse"}, args...) cmdArgs = append(cmdArgs, "a") - cmd.SetArgs(ConvertToPosixArgs(cmdArgs, cmd)) + cmd.SetArgs(convertToPosixArgs(cmdArgs, cmd)) if err = cmd.Execute(); err != nil { return nil, err } diff --git a/cmd/datatypes_parsing_test.go b/cmd/datatypes_parsing_test.go index b615c91a33..0561011f23 100644 --- a/cmd/datatypes_parsing_test.go +++ b/cmd/datatypes_parsing_test.go @@ -581,7 +581,7 @@ func TestCLIFlagPassing(t *testing.T) { require.NoError(t, err) cmdArgs := append([]string{"gcsfuse"}, tc.args...) cmdArgs = append(cmdArgs, "a") - command.SetArgs(ConvertToPosixArgs(cmdArgs, command)) + command.SetArgs(convertToPosixArgs(cmdArgs, command)) require.NoError(t, command.Execute()) @@ -753,7 +753,7 @@ func TestConfigPassing(t *testing.T) { }) require.NoError(t, err) cmdArgs := append([]string{"gcsfuse", fmt.Sprintf("--config-file=testdata/%s", tc.file)}, "a") - command.SetArgs(ConvertToPosixArgs(cmdArgs, command)) + command.SetArgs(convertToPosixArgs(cmdArgs, command)) require.NoError(t, command.Execute()) @@ -807,7 +807,7 @@ func TestPredefinedFlagThrowNoError(t *testing.T) { }) require.NoError(t, err) cmdArgs := append([]string{"gcsfuse"}, tc.args...) - command.SetArgs(ConvertToPosixArgs(cmdArgs, command)) + command.SetArgs(convertToPosixArgs(cmdArgs, command)) assert.NoError(t, command.Execute()) }) diff --git a/cmd/flags.go b/cmd/flags.go deleted file mode 100644 index a514685dbc..0000000000 --- a/cmd/flags.go +++ /dev/null @@ -1,677 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/common" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" - "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" - "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - mountpkg "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - "github.com/googlecloudplatform/gcsfuse/v2/internal/util" - "github.com/urfave/cli" -) - -const ( - // ExperimentalMetadataPrefetchOnMountFlag is the name of the commandline flag for enabling - // metadata-prefetch mode aka 'ls -R' during mount. - ExperimentalMetadataPrefetchOnMountFlag = "experimental-metadata-prefetch-on-mount" -) - -// Set up custom help text for gcsfuse; in particular the usage section. -func init() { - cli.AppHelpTemplate = `NAME: - {{.Name}} - {{.Usage}} - -USAGE: - {{.Name}} {{if .Flags}}[global options]{{end}} [bucket] mountpoint - {{if .Version}} -VERSION: - {{.Version}} - {{end}}{{if len .Authors}} -AUTHOR(S): - {{range .Authors}}{{ . }}{{end}} - {{end}}{{if .Flags}} -GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}}{{if .Copyright }} -COPYRIGHT: - {{.Copyright}} - {{end}} -` -} - -func newApp() (app *cli.App) { - dirModeValue := new(OctalInt) - *dirModeValue = 0755 - - fileModeValue := new(OctalInt) - *fileModeValue = 0644 - - app = &cli.App{ - Name: "gcsfuse", - Version: common.GetVersion(), - Usage: "Mount a specified GCS bucket or all accessible buckets locally", - Writer: os.Stderr, - Flags: []cli.Flag{ - - cli.StringFlag{ - Name: "app-name", - Value: "", - Usage: "The application name of this mount.", - }, - - cli.BoolFlag{ - Name: "foreground", - Usage: "Stay in the foreground after mounting.", - }, - - cli.StringFlag{ - Name: "config-file", - Value: "", - Usage: "The path to the config file where all gcsfuse related config needs to be specified. " + - "Refer to 'https://cloud.google.com/storage/docs/gcsfuse-cli#config-file' for possible configurations.", - }, - - ///////////////////////// - // File system - ///////////////////////// - - cli.StringSliceFlag{ - Name: "o", - Usage: "Additional system-specific mount options. Multiple options can be passed as comma separated. For readonly, use --o ro", - }, - - cli.GenericFlag{ - Name: "dir-mode", - Value: dirModeValue, - Usage: "Permissions bits for directories, in octal.", - }, - - cli.GenericFlag{ - Name: "file-mode", - Value: fileModeValue, - Usage: "Permission bits for files, in octal.", - }, - - cli.IntFlag{ - Name: "uid", - Value: -1, - Usage: "UID owner of all inodes.", - }, - - cli.IntFlag{ - Name: "gid", - Value: -1, - Usage: "GID owner of all inodes.", - }, - - cli.BoolFlag{ - Name: "implicit-dirs", - Usage: "Implicitly define directories based on content. See files and directories in docs/semantics for more information", - }, - - cli.StringFlag{ - Name: "only-dir", - Usage: "Mount only a specific directory within the bucket. See docs/mounting for more information", - }, - - cli.IntFlag{ - Name: "rename-dir-limit", - Value: 0, - Usage: "Allow rename a directory containing fewer descendants than this limit.", - }, - - cli.BoolTFlag{ - Name: "ignore-interrupts", - Usage: "Instructs gcsfuse to ignore system interrupt signals (like SIGINT, triggered by Ctrl+C). " + - "This prevents those signals from immediately terminating gcsfuse inflight operations. (default: true)", - }, - - ///////////////////////// - // GCS - ///////////////////////// - - cli.StringFlag{ - Name: "custom-endpoint", - Usage: "Specifies an alternative custom endpoint for fetching data. Should only be used for testing. " + - "The custom endpoint must support the equivalent resources and operations as the GCS " + - "JSON endpoint, https://storage.googleapis.com/storage/v1. If a custom endpoint is not specified, " + - "GCSFuse uses the global GCS JSON API endpoint, https://storage.googleapis.com/storage/v1.", - }, - - cli.BoolFlag{ - Name: "anonymous-access", - Usage: "Authentication is enabled by default. This flag will disable authentication", - }, - - cli.StringFlag{ - Name: "billing-project", - Value: "", - Usage: "Project to use for billing when accessing a bucket enabled with “Requester Pays” (default: none)", - }, - - cli.StringFlag{ - Name: "key-file", - Value: "", - Usage: "Absolute path to JSON key file for use with GCS. (default: none, Google application default credentials used)", - }, - - cli.StringFlag{ - Name: "token-url", - Value: "", - Usage: "A url for getting an access token when the key-file is absent.", - }, - - cli.BoolTFlag{ - Name: "reuse-token-from-url", - Usage: "If false, the token acquired from token-url is not reused.", - }, - - cli.Float64Flag{ - Name: "limit-bytes-per-sec", - Value: -1, - Usage: "Bandwidth limit for reading data, measured over a 30-second window. (use -1 for no limit)", - }, - - cli.Float64Flag{ - Name: "limit-ops-per-sec", - Value: -1, - Usage: "Operations per second limit, measured over a 30-second window (use -1 for no limit)", - }, - - cli.IntFlag{ - Name: "sequential-read-size-mb", - Value: 200, - Usage: "File chunk size to read from GCS in one call. Need to specify the value in MB. ChunkSize less than 1MB is not supported", - }, - - ///////////////////////// - // Tuning - ///////////////////////// - - cli.DurationFlag{ - Name: "max-retry-sleep", - Value: 30 * time.Second, - Usage: "The maximum duration allowed to sleep in a retry loop with exponential backoff for failed requests to GCS backend." + - " Once the backoff duration exceeds this limit, the retry continues with this specified maximum value.", - }, - - cli.IntFlag{ - Name: "max-retry-attempts", - Value: 0, - Usage: "The max-retry-attempts parameter sets a limit on the number of times an operation will be retried if it fails, preventing endless retry loops. The default value 0 indicates no limit", - }, - - cli.IntFlag{ - Name: "stat-cache-capacity", - Value: mount.DefaultStatCacheCapacity, - Usage: "How many entries can the stat-cache hold (impacts memory consumption). This flag has been deprecated (starting v2.0) and in its place only metadata-cache:stat-cache-max-size-mb in the gcsfuse config-file will be supported. For now, the value of stat-cache-capacity will be translated to the next higher corresponding value of metadata-cache:stat-cache-max-size-mb (assuming stat-cache entry-size ~= 1640 bytes, including 1400 for positive entry and 240 for corresponding negative entry), when metadata-cache:stat-cache-max-size-mb is not set.", - }, - - cli.DurationFlag{ - Name: "stat-cache-ttl", - Value: mount.DefaultStatOrTypeCacheTTL, - Usage: "How long to cache StatObject results and inode attributes. This flag has been deprecated (starting v2.0) and in its place only metadata-cache:ttl-secs in the gcsfuse config-file will be supported. For now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to the next higher multiple of a second, is used as ttl for both stat-cache and type-cache, when metadata-cache:ttl-secs is not set.", - }, - - cli.DurationFlag{ - Name: "type-cache-ttl", - Value: mount.DefaultStatOrTypeCacheTTL, - Usage: "How long to cache name -> file/dir mappings in directory inodes. This flag has been deprecated (starting v2.0) and in its place only metadata-cache:ttl-secs in the gcsfuse config-file will be supported. For now, the minimum of stat-cache-ttl and type-cache-ttl values, rounded up to the next higher multiple of a second, is used as ttl for both stat-cache and type-cache, when metadata-cache:ttl-secs is not set.", - }, - - cli.Int64Flag{ - Name: "kernel-list-cache-ttl-secs", - Value: config.DefaultKernelListCacheTtlSeconds, - Usage: "How long the directory listing (output of ls ) should be cached in the kernel page cache." + - "If a particular directory cache entry is kept by kernel for longer than TTL, then it will be sent for invalidation " + - "by gcsfuse on next opendir (comes in the start, as part of next listing) call. 0 means no caching. " + - "Use -1 to cache for lifetime (no ttl). Negative value other than -1 will throw error.", - }, - - cli.DurationFlag{ - Name: "http-client-timeout", - Usage: "The time duration that http client will wait to get response from the server. The default value 0 indicates no timeout. ", - }, - - cli.DurationFlag{ - Name: "max-retry-duration", - Value: -1 * time.Second, - Usage: "This flag is currently unused.", - }, - - cli.Float64Flag{ - Name: "retry-multiplier", - Value: 2, - Usage: "Param for exponential backoff algorithm, which is used to increase waiting time b/w two consecutive retries.", - }, - - cli.StringFlag{ - Name: "temp-dir", - Value: "", - Usage: "Path to the temporary directory where writes are staged prior to" + - " upload to Cloud Storage. (default: system default, likely /tmp)", - }, - - cli.StringFlag{ - Name: "client-protocol", - Value: string(mountpkg.HTTP1), - Usage: "The protocol used for communicating with the GCS backend. " + - "Value can be 'http1' (HTTP/1.1) or 'http2' (HTTP/2) or grpc.", - }, - - cli.IntFlag{ - Name: "max-conns-per-host", - Value: 0, - Usage: "The max number of TCP connections allowed per server. This is " + - "effective when --client-protocol is set to 'http1'. The default value" + - " 0 indicates no limit on TCP connections (limited by the machine specifications)", - }, - - cli.IntFlag{ - Name: "max-idle-conns-per-host", - Value: 100, - Usage: "The number of maximum idle connections allowed per server.", - }, - - cli.BoolFlag{ - Name: "enable-nonexistent-type-cache", - Usage: "Once set, if an inode is not found in GCS, a type cache entry with type NonexistentType" + - " will be created. This also means new file/dir created might not be seen. For example, if this" + - " flag is set, and metadata-cache:ttl-secs (in config-file) or flag type-cache-ttl are set, then if we create the same file/node" + - " in the meantime using the same mount, since we are not refreshing the cache, it will still return nil.", - }, - - ///////////////////////// - // Monitoring & Logging - ///////////////////////// - - cli.DurationFlag{ - Name: "stackdriver-export-interval", - Value: 0, - Usage: "Export metrics to stackdriver with this interval. The default value 0 indicates no exporting.", - }, - - cli.StringFlag{ - Name: "experimental-opentelemetry-collector-address", - Value: "", - Usage: "Experimental: Export metrics to the OpenTelemetry collector at this address.", - }, - - cli.IntFlag{ - Name: "prometheus-port", - Value: 0, - Usage: "Expose Prometheus metrics endpoint on this port and a path of /metrics.", - }, - - cli.StringFlag{ - Name: "log-file", - Value: "", - Usage: "The file for storing logs that can be parsed by fluentd. When not provided," + - " plain text logs are printed to stdout.", - }, - - cli.StringFlag{ - Name: "log-format", - Value: "json", - Usage: "The format of the log file: 'text' or 'json'.", - }, - - cli.BoolFlag{ - Name: "experimental-enable-json-read", - Usage: "By default, GCSFuse uses the GCS XML API to get and read objects. " + - "When this flag is specified, GCSFuse uses the GCS JSON API instead.", - }, - - ///////////////////////// - // Debugging - ///////////////////////// - - cli.BoolTFlag{ - Name: "debug_fuse_errors", - Usage: "If false, fuse errors will not be logged to the console (in case of --foreground) " + - "or the log-file (if specified)", - }, - - cli.BoolFlag{ - Name: "debug_fuse", - Usage: "Enable fuse-related debugging output.", - }, - - cli.BoolFlag{ - Name: "debug_fs", - Usage: "This flag is currently unused.", - }, - - cli.BoolFlag{ - Name: "debug_gcs", - Usage: "Print GCS request and timing information.", - }, - - cli.BoolFlag{ - Name: "debug_http", - Usage: "This flag is currently unused.", - }, - - cli.BoolFlag{ - Name: "debug_invariants", - Usage: "Panic when internal invariants are violated.", - }, - - cli.BoolFlag{ - Name: "debug_mutex", - Usage: "Print debug messages when a mutex is held too long.", - }, - - ///////////////////////// - // Post-mount actions - ///////////////////////// - - cli.StringFlag{ - Name: ExperimentalMetadataPrefetchOnMountFlag, - Value: cfg.ExperimentalMetadataPrefetchOnMountDisabled, - Usage: "Experimental: This indicates whether or not to prefetch the metadata (prefilling of metadata caches and creation of inodes) of the mounted bucket at the time of mounting the bucket. Supported values: \"disabled\", \"sync\" and \"async\". Any other values will return error on mounting. This is applicable only to static mounting, and not to dynamic mounting.", - }, - }, - } - - return -} - -type flagStorage struct { - // Deprecated: Use the param from cfg/config.go - AppName string - - // Deprecated: Use the param from cfg/config.go - Foreground bool - ConfigFile string - - // File system - // Deprecated: Use the param from cfg/config.go - MountOptions []string - - // Deprecated: Use the param from cfg/config.go - DirMode os.FileMode - - // Deprecated: Use the param from cfg/config.go - FileMode os.FileMode - - // Deprecated: Use the param from cfg/config.go - Uid int64 - - // Deprecated: Use the param from cfg/config.go - Gid int64 - - // Deprecated: Use the param from cfg/config.go - ImplicitDirs bool - - // Deprecated: Use the param from cfg/config.go - OnlyDir string - - // Deprecated: Use the param from cfg/config.go - RenameDirLimit int64 - // Deprecated: Use the param from cfg/config.go - IgnoreInterrupts bool - - // GCS - // Deprecated: Use the param from cfg/config.go - CustomEndpoint *url.URL - - // Deprecated: Use the param from cfg/config.go - BillingProject string - - // Deprecated: Use the param from cfg/config.go - KeyFile string - - // Deprecated: Use the param from cfg/config.go - TokenUrl string - - // Deprecated: Use the param from cfg/config.go - ReuseTokenFromUrl bool - - // Deprecated: Use the param from cfg/config.go - EgressBandwidthLimitBytesPerSecond float64 - - // Deprecated: Use the param from cfg/config.go - OpRateLimitHz float64 - // Deprecated: Use the param from cfg/config.go - SequentialReadSizeMb int32 - // Deprecated: Use the param from cfg/config.go - AnonymousAccess bool - - // Tuning - // Deprecated: Use the param from cfg/config.go - MaxRetrySleep time.Duration - - // Deprecated: Use the param from cfg/config.go - MaxRetryAttempts int64 - - // Deprecated: Use the param from cfg/config.go - StatCacheCapacity int - - // Deprecated: Use the param from cfg/config.go - StatCacheTTL time.Duration - - // Deprecated: Use the param from cfg/config.go - TypeCacheTTL time.Duration - KernelListCacheTtlSeconds int64 - - // Deprecated: Use the param from cfg/config.go - HttpClientTimeout time.Duration - - // Deprecated: Use the param from cfg/config.go - RetryMultiplier float64 - - // Deprecated: Use the param from cfg/config.go - TempDir string - - // Deprecated: Use the param from cfg/config.go - ClientProtocol mountpkg.ClientProtocol - - // Deprecated: Use the param from cfg/config.go - MaxConnsPerHost int - - // Deprecated: Use the param from cfg/config.go - MaxIdleConnsPerHost int - - // Deprecated: Use the param from cfg/config.go - EnableNonexistentTypeCache bool - - // Monitoring & Logging - // Deprecated: Use the param from cfg/config.go - StackdriverExportInterval time.Duration - PrometheusPort int - - // Deprecated: Use the param from cfg/config.go - OtelCollectorAddress string - - // Deprecated: Use the param from cfg/config.go - LogFile string - - // Deprecated: Use the param from cfg/config.go - LogFormat string - - // Deprecated: Use the param from cfg/config.go - ExperimentalEnableJsonRead bool - - // Debugging - - // Deprecated: Use the param from cfg/config.go - DebugFuse bool - - // Deprecated: Use the param from cfg/config.go - DebugGCS bool - - // Deprecated: Use the param from cfg/config.go - DebugInvariants bool - - // Deprecated: Use the param from cfg/config.go - DebugMutex bool - - // Post-mount actions - - // Deprecated: ExperimentalMetadataPrefetchOnMount indicates whether or not to prefetch the metadata of the mounted bucket at the time of mounting the bucket. - // Supported values: ExperimentalMetadataPrefetchOnMountDisabled, ExperimentalMetadataPrefetchOnMountSynchronous, and ExperimentalMetadataPrefetchOnMountAsynchronous. - // Any other values will return error on mounting. - // This is applicable only to single-bucket mount-points, and not to dynamic-mount points. This is because dynamic-mounts don't mount the bucket(s) at the time of - // gcsfuse command itself, which flag is targeted at. - ExperimentalMetadataPrefetchOnMount string -} - -func resolveFilePath(filePath string, configKey string) (resolvedPath string, err error) { - resolvedPath, err = util.GetResolvedPath(filePath) - if filePath == resolvedPath || err != nil { - return - } - - logger.Infof("Value of [%s] resolved from [%s] to [%s]\n", configKey, filePath, resolvedPath) - return resolvedPath, nil -} - -// This method resolves path in the context dictionary. -func resolvePathForTheFlagInContext(flagKey string, c *cli.Context) (err error) { - flagValue := c.String(flagKey) - resolvedPath, err := resolveFilePath(flagValue, flagKey) - if err != nil { - return - } - - err = c.Set(flagKey, resolvedPath) - return -} - -// For parent process: it only resolves the path with respect to home folder. -// For child process: it resolves the path relative to both home directory and -// GCSFUSE_PARENT_PROCESS_DIR. Child process is spawned when --foreground flag -// is disabled. -func resolvePathForTheFlagsInContext(c *cli.Context) (err error) { - err = resolvePathForTheFlagInContext("config-file", c) - if err != nil { - return fmt.Errorf("resolving for config-file: %w", err) - } - - return -} - -// Add the flags accepted by run to the supplied flag set, returning the -// variables into which the flags will parse. -func populateFlags(c *cli.Context) (flags *flagStorage, err error) { - customEndpointStr := c.String("custom-endpoint") - var customEndpoint *url.URL - - if customEndpointStr == "" { - customEndpoint = nil - } else { - customEndpoint, err = url.Parse(customEndpointStr) - if customEndpoint.String() == "" || err != nil { - err = fmt.Errorf("could not parse custom-endpoint: %w", err) - return - } - } - - clientProtocolString := strings.ToLower(c.String("client-protocol")) - clientProtocol := mountpkg.ClientProtocol(clientProtocolString) - flags = &flagStorage{ - AppName: c.String("app-name"), - Foreground: c.Bool("foreground"), - ConfigFile: c.String("config-file"), - - // File system - MountOptions: c.StringSlice("o"), - DirMode: os.FileMode(*c.Generic("dir-mode").(*OctalInt)), - FileMode: os.FileMode(*c.Generic("file-mode").(*OctalInt)), - Uid: int64(c.Int("uid")), - Gid: int64(c.Int("gid")), - ImplicitDirs: c.Bool("implicit-dirs"), - OnlyDir: c.String("only-dir"), - RenameDirLimit: int64(c.Int("rename-dir-limit")), - IgnoreInterrupts: c.Bool("ignore-interrupts"), - - // GCS, - CustomEndpoint: customEndpoint, - AnonymousAccess: c.Bool("anonymous-access"), - BillingProject: c.String("billing-project"), - KeyFile: c.String("key-file"), - TokenUrl: c.String("token-url"), - ReuseTokenFromUrl: c.BoolT("reuse-token-from-url"), - EgressBandwidthLimitBytesPerSecond: c.Float64("limit-bytes-per-sec"), - OpRateLimitHz: c.Float64("limit-ops-per-sec"), - SequentialReadSizeMb: int32(c.Int("sequential-read-size-mb")), - - // Tuning, - MaxRetrySleep: c.Duration("max-retry-sleep"), - MaxRetryAttempts: c.Int64("max-retry-attempts"), - StatCacheCapacity: c.Int("stat-cache-capacity"), - StatCacheTTL: c.Duration("stat-cache-ttl"), - TypeCacheTTL: c.Duration("type-cache-ttl"), - KernelListCacheTtlSeconds: c.Int64("kernel-list-cache-ttl-secs"), - HttpClientTimeout: c.Duration("http-client-timeout"), - RetryMultiplier: c.Float64("retry-multiplier"), - TempDir: c.String("temp-dir"), - ClientProtocol: clientProtocol, - MaxConnsPerHost: c.Int("max-conns-per-host"), - MaxIdleConnsPerHost: c.Int("max-idle-conns-per-host"), - EnableNonexistentTypeCache: c.Bool("enable-nonexistent-type-cache"), - - // Monitoring & Logging - StackdriverExportInterval: c.Duration("stackdriver-export-interval"), - OtelCollectorAddress: c.String("experimental-opentelemetry-collector-address"), - PrometheusPort: c.Int("prometheus-port"), - LogFile: c.String("log-file"), - LogFormat: c.String("log-format"), - ExperimentalEnableJsonRead: c.Bool("experimental-enable-json-read"), - - // Debugging, - DebugFuse: c.Bool("debug_fuse"), - DebugGCS: c.Bool("debug_gcs"), - DebugInvariants: c.Bool("debug_invariants"), - DebugMutex: c.Bool("debug_mutex"), - - // Post-mount actions - ExperimentalMetadataPrefetchOnMount: c.String(ExperimentalMetadataPrefetchOnMountFlag), - } - - return -} - -// A cli.Generic that can be used with cli.GenericFlag to obtain an int flag -// that is parsed in octal. -type OctalInt int - -var _ cli.Generic = (*OctalInt)(nil) - -func (oi *OctalInt) Set(value string) (err error) { - tmp, err := strconv.ParseInt(value, 8, 32) - if err != nil { - err = fmt.Errorf("parsing as octal: %w", err) - return - } - - *oi = OctalInt(tmp) - return -} - -func (oi OctalInt) String() string { - return fmt.Sprintf("%o", oi) -} diff --git a/cmd/flags_test.go b/cmd/flags_test.go deleted file mode 100644 index a104d54a1f..0000000000 --- a/cmd/flags_test.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "os" - "path/filepath" - "sort" - "testing" - "time" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" - "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - mountpkg "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "github.com/urfave/cli" -) - -func TestFlags(t *testing.T) { suite.Run(t, new(FlagsTest)) } - -//////////////////////////////////////////////////////////////////////// -// Boilerplate -//////////////////////////////////////////////////////////////////////// - -type FlagsTest struct { - suite.Suite -} - -func parseArgs(t *FlagsTest, args []string) (flags *flagStorage) { - // Create a CLI app, and abuse it to snoop on the flags. - app := newApp() - var err error - app.Action = func(appCtx *cli.Context) { - flags, err = populateFlags(appCtx) - assert.Equal(t.T(), nil, err) - } - - // Simulate argv. - fullArgs := append([]string{"some_app"}, args...) - - err = app.Run(fullArgs) - assert.Equal(t.T(), nil, err) - - return -} - -//////////////////////////////////////////////////////////////////////// -// Tests -//////////////////////////////////////////////////////////////////////// - -func (t *FlagsTest) TestDefaults() { - f := parseArgs(t, []string{}) - - // File system - assert.NotEqual(t.T(), nil, f.MountOptions) - assert.Equal(t.T(), 0, len(f.MountOptions), "Options: %v", f.MountOptions) - - assert.Equal(t.T(), os.FileMode(0755), f.DirMode) - assert.Equal(t.T(), os.FileMode(0644), f.FileMode) - assert.EqualValues(t.T(), -1, f.Uid) - assert.EqualValues(t.T(), -1, f.Gid) - assert.False(t.T(), f.ImplicitDirs) - assert.True(t.T(), f.IgnoreInterrupts) - assert.Equal(t.T(), config.DefaultKernelListCacheTtlSeconds, f.KernelListCacheTtlSeconds) - - // GCS - assert.Equal(t.T(), "", f.KeyFile) - assert.EqualValues(t.T(), -1, f.EgressBandwidthLimitBytesPerSecond) - assert.EqualValues(t.T(), -1, f.OpRateLimitHz) - assert.True(t.T(), f.ReuseTokenFromUrl) - assert.Nil(t.T(), f.CustomEndpoint) - assert.False(t.T(), f.AnonymousAccess) - - // Tuning - assert.Equal(t.T(), mount.DefaultStatCacheCapacity, f.StatCacheCapacity) - assert.Equal(t.T(), mount.DefaultStatOrTypeCacheTTL, f.StatCacheTTL) - assert.Equal(t.T(), mount.DefaultStatOrTypeCacheTTL, f.TypeCacheTTL) - assert.EqualValues(t.T(), 0, f.HttpClientTimeout) - assert.Equal(t.T(), "", f.TempDir) - assert.Equal(t.T(), config.DefaultMaxRetryAttempts, f.MaxRetryAttempts) - assert.EqualValues(t.T(), 2, f.RetryMultiplier) - assert.False(t.T(), f.EnableNonexistentTypeCache) - assert.Equal(t.T(), 0, f.MaxConnsPerHost) - - // Debugging - assert.False(t.T(), f.DebugFuse) - assert.False(t.T(), f.DebugGCS) - assert.False(t.T(), f.DebugInvariants) - - // Post-mount actions - assert.Equal(t.T(), cfg.ExperimentalMetadataPrefetchOnMountDisabled, f.ExperimentalMetadataPrefetchOnMount) - - // Metrics - assert.Equal(t.T(), 0, f.PrometheusPort) -} - -func (t *FlagsTest) TestBools() { - names := []string{ - "implicit-dirs", - "reuse-token-from-url", - "debug_fuse_errors", - "debug_fuse", - "debug_http", - "debug_gcs", - "debug_invariants", - "enable-nonexistent-type-cache", - "experimental-enable-json-read", - "ignore-interrupts", - "anonymous-access", - } - - var args []string - var f *flagStorage - - // --foo form - args = nil - for _, n := range names { - args = append(args, fmt.Sprintf("-%s", n)) - } - - f = parseArgs(t, args) - assert.True(t.T(), f.ImplicitDirs) - assert.True(t.T(), f.ReuseTokenFromUrl) - assert.True(t.T(), f.DebugFuse) - assert.True(t.T(), f.DebugGCS) - assert.True(t.T(), f.DebugInvariants) - assert.True(t.T(), f.EnableNonexistentTypeCache) - assert.True(t.T(), f.ExperimentalEnableJsonRead) - assert.True(t.T(), f.IgnoreInterrupts) - assert.True(t.T(), f.AnonymousAccess) - - // --foo=false form - args = nil - for _, n := range names { - args = append(args, fmt.Sprintf("-%s=false", n)) - } - - f = parseArgs(t, args) - assert.False(t.T(), f.ImplicitDirs) - assert.False(t.T(), f.ReuseTokenFromUrl) - assert.False(t.T(), f.DebugFuse) - assert.False(t.T(), f.DebugGCS) - assert.False(t.T(), f.DebugInvariants) - assert.False(t.T(), f.EnableNonexistentTypeCache) - - // --foo=true form - args = nil - for _, n := range names { - args = append(args, fmt.Sprintf("-%s=true", n)) - } - - f = parseArgs(t, args) - assert.True(t.T(), f.ImplicitDirs) - assert.True(t.T(), f.ReuseTokenFromUrl) - assert.True(t.T(), f.DebugFuse) - assert.True(t.T(), f.DebugGCS) - assert.True(t.T(), f.DebugInvariants) - assert.True(t.T(), f.EnableNonexistentTypeCache) -} - -func (t *FlagsTest) TestDecimalNumbers() { - args := []string{ - "--uid=17", - "--gid=19", - "--limit-bytes-per-sec=123.4", - "--limit-ops-per-sec=56.78", - "--stat-cache-capacity=8192", - "--max-idle-conns-per-host=100", - "--max-conns-per-host=100", - "--kernel-list-cache-ttl-secs=234", - "--max-retry-attempts=100", - } - - f := parseArgs(t, args) - assert.EqualValues(t.T(), 17, f.Uid) - assert.EqualValues(t.T(), 19, f.Gid) - assert.Equal(t.T(), 123.4, f.EgressBandwidthLimitBytesPerSecond) - assert.Equal(t.T(), 56.78, f.OpRateLimitHz) - assert.Equal(t.T(), 8192, f.StatCacheCapacity) - assert.Equal(t.T(), 100, f.MaxIdleConnsPerHost) - assert.Equal(t.T(), 100, f.MaxConnsPerHost) - assert.EqualValues(t.T(), 234, f.KernelListCacheTtlSeconds) - assert.EqualValues(t.T(), 100, f.MaxRetryAttempts) -} - -func (t *FlagsTest) TestOctalNumbers() { - args := []string{ - "--dir-mode=711", - "--file-mode", "611", - } - - f := parseArgs(t, args) - assert.Equal(t.T(), os.FileMode(0711), f.DirMode) - assert.Equal(t.T(), os.FileMode(0611), f.FileMode) -} - -func (t *FlagsTest) TestStrings() { - args := []string{ - "--key-file", "-asdf", - "--temp-dir=foobar", - "--only-dir=baz", - "--client-protocol=HTTP2", - "--experimental-metadata-prefetch-on-mount=async", - } - - f := parseArgs(t, args) - assert.Equal(t.T(), "-asdf", f.KeyFile) - assert.Equal(t.T(), "foobar", f.TempDir) - assert.Equal(t.T(), "baz", f.OnlyDir) - assert.Equal(t.T(), mountpkg.HTTP2, f.ClientProtocol) - assert.Equal(t.T(), cfg.ExperimentalMetadataPrefetchOnMountAsynchronous, f.ExperimentalMetadataPrefetchOnMount) -} - -func (t *FlagsTest) TestDurations() { - args := []string{ - "--stat-cache-ttl", "1m17s100ms", - "--type-cache-ttl", "50s900ms", - "--http-client-timeout", "800ms", - "--max-retry-duration", "-1s", - "--max-retry-sleep", "30s", - } - - f := parseArgs(t, args) - assert.Equal(t.T(), time.Minute+17*time.Second+100*time.Millisecond, f.StatCacheTTL) - assert.Equal(t.T(), 50*time.Second+900*time.Millisecond, f.TypeCacheTTL) - assert.Equal(t.T(), 800*time.Millisecond, f.HttpClientTimeout) - assert.Equal(t.T(), 30*time.Second, f.MaxRetrySleep) -} - -func (t *FlagsTest) TestSlice() { - args := []string{ - "-o", "rw,nodev", - "-o", "user=jacobsa,noauto", - } - - f := parseArgs(t, args) - - sort.Strings(f.MountOptions) - require.Equal(t.T(), 2, len(f.MountOptions)) - assert.Equal(t.T(), "rw,nodev", f.MountOptions[0]) - assert.Equal(t.T(), "user=jacobsa,noauto", f.MountOptions[1]) -} - -func (t *FlagsTest) TestResolvePathForTheFlagInContext() { - app := newApp() - currentWorkingDir, err := os.Getwd() - assert.Equal(t.T(), nil, err) - app.Action = func(appCtx *cli.Context) { - err = resolvePathForTheFlagInContext("key-file", appCtx) - assert.Equal(t.T(), nil, err) - err = resolvePathForTheFlagInContext("config-file", appCtx) - assert.Equal(t.T(), nil, err) - - assert.Equal(t.T(), filepath.Join(currentWorkingDir, "test.txt"), - appCtx.String("key-file")) - assert.Equal(t.T(), filepath.Join(currentWorkingDir, "config.yaml"), - appCtx.String("config-file")) - } - // Simulate argv. - fullArgs := []string{"some_app", "--key-file=test.txt", "--config-file=config.yaml"} - - err = app.Run(fullArgs) - - assert.Equal(t.T(), nil, err) -} - -func (t *FlagsTest) TestResolvePathForTheFlagsInContext() { - app := newApp() - currentWorkingDir, err := os.Getwd() - assert.Equal(t.T(), nil, err) - app.Action = func(appCtx *cli.Context) { - resolvePathForTheFlagsInContext(appCtx) - - assert.Equal(t.T(), filepath.Join(currentWorkingDir, "config.yaml"), - appCtx.String("config-file")) - } - // Simulate argv. - fullArgs := []string{"some_app", "--config-file=config.yaml"} - - err = app.Run(fullArgs) - - assert.Equal(t.T(), nil, err) -} - -func (t *FlagsTest) Test_KernelListCacheTtlSecs() { - args := []string{ - "--kernel-list-cache-ttl-secs=-1", - } - - f := parseArgs(t, args) - - assert.Equal(t.T(), int64(-1), f.KernelListCacheTtlSeconds) -} - -func (t *FlagsTest) Test_KernelListCacheTtlSecs_MaxValid() { - args := []string{ - "--kernel-list-cache-ttl-secs=9223372036", - } - - f := parseArgs(t, args) - - assert.Equal(t.T(), int64(9223372036), f.KernelListCacheTtlSeconds) -} - -func (t *FlagsTest) Test_PrometheusPort() { - args := []string{ - "--prometheus-port=8080", - } - - f := parseArgs(t, args) - - assert.Equal(t.T(), 8080, f.PrometheusPort) -} diff --git a/cmd/legacy_main.go b/cmd/legacy_main.go index 7a228c5df8..f713b2470b 100644 --- a/cmd/legacy_main.go +++ b/cmd/legacy_main.go @@ -31,7 +31,6 @@ import ( "github.com/googlecloudplatform/gcsfuse/v2/cfg" "github.com/googlecloudplatform/gcsfuse/v2/common" "github.com/googlecloudplatform/gcsfuse/v2/internal/canned" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/internal/locker" "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" "github.com/googlecloudplatform/gcsfuse/v2/internal/monitor" @@ -42,14 +41,12 @@ import ( "github.com/jacobsa/daemonize" "github.com/jacobsa/fuse" "github.com/kardianos/osext" - "github.com/urfave/cli" "golang.org/x/net/context" ) const ( SuccessfulMountMessage = "File system has been successfully mounted." UnsuccessfulMountMessagePrefix = "Error while mounting gcsfuse" - EnableViperConfigEnvVariable = "ENABLE_GCSFUSE_VIPER_CONFIG" ) //////////////////////////////////////////////////////////////////////// @@ -237,33 +234,6 @@ func isDynamicMount(bucketName string) bool { return bucketName == "" || bucketName == "_" } -func runCLIApp(c *cli.Context) (err error) { - err = resolvePathForTheFlagsInContext(c) - if err != nil { - return fmt.Errorf("resolving path: %w", err) - } - - flags, err := populateFlags(c) - if err != nil { - return fmt.Errorf("parsing flags failed: %w", err) - } - - mountConfig, err := config.ParseConfigFile(flags.ConfigFile) - if err != nil { - return fmt.Errorf("parsing config file failed: %w", err) - } - - newConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(c, flags, mountConfig) - if err != nil { - return fmt.Errorf("error resolving flags and configs: %w", err) - } - var bucketName, mountPoint string - if bucketName, mountPoint, err = populateArgs(c.Args()); err != nil { - return err - } - return Mount(newConfig, bucketName, mountPoint) -} - func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { // Ideally this call to SetLogFormat (which internally creates a new defaultLogger) // should be set as an else to the 'if flags.Foreground' check below, but currently @@ -320,10 +290,6 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { fmt.Sprintf("PATH=%s", os.Getenv("PATH")), } - // Pass along ENABLE_GCSFUSE_VIPER_CONFIG environment variable, since we - // need to use Viper config if this environment variable is set. - env = append(env, fmt.Sprintf(EnableViperConfigEnvVariable+"=%s", os.Getenv(EnableViperConfigEnvVariable))) - // Pass along GOOGLE_APPLICATION_CREDENTIALS, since we document in // mounting.md that it can be used for specifying a key file. if p, ok := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); ok { @@ -473,30 +439,3 @@ func Mount(newConfig *cfg.Config, bucketName, mountPoint string) (err error) { return } - -func run() (err error) { - // Set up the app. - app := newApp() - - var appErr error - app.Action = func(c *cli.Context) { - appErr = runCLIApp(c) - } - - // Run it. - err = app.Run(os.Args) - if err != nil { - return - } - - err = appErr - return -} - -var ExecuteLegacyMain = func() { - err := run() - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} diff --git a/cmd/legacy_param_mapper.go b/cmd/legacy_param_mapper.go deleted file mode 100644 index 4ce1bdd358..0000000000 --- a/cmd/legacy_param_mapper.go +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "reflect" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" - "github.com/mitchellh/mapstructure" -) - -// cliContext is abstraction over the IsSet() method of cli.Context, specially -// added to keep OverrideWithIgnoreInterruptsFlag method's unit test simple. -type cliContext interface { - IsSet(string) bool -} - -// PopulateNewConfigFromLegacyFlagsAndConfig takes cliContext, legacy flags and legacy MountConfig and resolves it into new cfg.Config Object. -func PopulateNewConfigFromLegacyFlagsAndConfig(c cliContext, flags *flagStorage, legacyConfig *config.MountConfig) (*cfg.Config, error) { - if flags == nil || legacyConfig == nil { - return nil, fmt.Errorf("PopulateNewConfigFromLegacyFlagsAndConfig: unexpected nil flags or mount config") - } - - // Resolve custom endpoint url type to string. - var customEndPoint string - if flags.CustomEndpoint != nil { - customEndPoint = flags.CustomEndpoint.String() - } - - resolvedConfig := &cfg.Config{} - - structuredFlags := &map[string]interface{}{ - "app-name": flags.AppName, - "debug": &map[string]interface{}{ - "exit-on-invariant-violation": flags.DebugInvariants, - "gcs": flags.DebugGCS, - "log-mutex": flags.DebugMutex, - "fuse": flags.DebugFuse, - }, - "file-system": map[string]interface{}{ - "dir-mode": flags.DirMode, - "file-mode": flags.FileMode, - "fuse-options": flags.MountOptions, - "gid": flags.Gid, - "ignore-interrupts": flags.IgnoreInterrupts, - "rename-dir-limit": flags.RenameDirLimit, - "temp-dir": flags.TempDir, - "uid": flags.Uid, - "kernel-list-cache-ttl-secs": flags.KernelListCacheTtlSeconds, - }, - "foreground": flags.Foreground, - "gcs-auth": map[string]interface{}{ - "anonymous-access": flags.AnonymousAccess, - "key-file": flags.KeyFile, - "reuse-token-from-url": flags.ReuseTokenFromUrl, - "token-url": flags.TokenUrl, - }, - "gcs-connection": map[string]interface{}{ - "billing-project": flags.BillingProject, - "client-protocol": string(flags.ClientProtocol), - "custom-endpoint": customEndPoint, - "experimental-enable-json-read": flags.ExperimentalEnableJsonRead, - "http-client-timeout": flags.HttpClientTimeout, - "limit-bytes-per-sec": flags.EgressBandwidthLimitBytesPerSecond, - "limit-ops-per-sec": flags.OpRateLimitHz, - "max-conns-per-host": flags.MaxConnsPerHost, - "max-idle-conns-per-host": flags.MaxIdleConnsPerHost, - "sequential-read-size-mb": flags.SequentialReadSizeMb, - }, - "gcs-retries": map[string]interface{}{ - "max-retry-sleep": flags.MaxRetrySleep, - "multiplier": flags.RetryMultiplier, - "max-retry-attempts": flags.MaxRetryAttempts, - }, - "implicit-dirs": flags.ImplicitDirs, - "logging": map[string]interface{}{ - "file-path": flags.LogFile, - "format": flags.LogFormat, - }, - "metadata-cache": map[string]interface{}{ - "deprecated-stat-cache-capacity": flags.StatCacheCapacity, - "deprecated-stat-cache-ttl": flags.StatCacheTTL, - "deprecated-type-cache-ttl": flags.TypeCacheTTL, - "enable-nonexistent-type-cache": flags.EnableNonexistentTypeCache, - "experimental-metadata-prefetch-on-mount": flags.ExperimentalMetadataPrefetchOnMount, - }, - "metrics": map[string]interface{}{ - "prometheus-port": flags.PrometheusPort, - "stackdriver-export-interval": flags.StackdriverExportInterval, - }, - "monitoring": map[string]interface{}{ - "experimental-opentelemetry-collector-address": flags.OtelCollectorAddress, - }, - "only-dir": flags.OnlyDir, - } - - // Use decoder to convert flagStorage to cfg.Config. - decoderConfig := &mapstructure.DecoderConfig{ - DecodeHook: cfg.DecodeHook(), - Result: resolvedConfig, - TagName: "yaml", - } - decoder, err := mapstructure.NewDecoder(decoderConfig) - if err != nil { - return nil, fmt.Errorf("mapstructure.NewDecoder: %w", err) - } - // Decoding flags. - if err = decoder.Decode(structuredFlags); err != nil { - return nil, fmt.Errorf("decoder.Decode(structuredFlags): %w", err) - } - - // If config file does not have any values, no need to decode it. - if reflect.ValueOf(*legacyConfig).IsZero() { - return resolvedConfig, nil - } - - // Save overlapping flags in a map to override the config value later. - var ( - logFile = resolvedConfig.Logging.FilePath - logFormat = resolvedConfig.Logging.Format - ignoreInterrupts = resolvedConfig.FileSystem.IgnoreInterrupts - anonymousAccess = resolvedConfig.GcsAuth.AnonymousAccess - kernelListCacheTTLSecs = resolvedConfig.FileSystem.KernelListCacheTtlSecs - maxRetryAttempts = resolvedConfig.GcsRetries.MaxRetryAttempts - prometheusPort = resolvedConfig.Metrics.PrometheusPort - ) - - // Decoding config to the same config structure (resolvedConfig). - if err = decoder.Decode(legacyConfig); err != nil { - return nil, fmt.Errorf("decoder.Decode(config): %w", err) - } - - // Override/Give priority to flags in case of overlap in flags and config. - overrideWithFlag(c, "log-file", &resolvedConfig.Logging.FilePath, logFile) - overrideWithFlag(c, "log-format", &resolvedConfig.Logging.Format, logFormat) - overrideWithFlag(c, "ignore-interrupts", &resolvedConfig.FileSystem.IgnoreInterrupts, ignoreInterrupts) - overrideWithFlag(c, "anonymous-access", &resolvedConfig.GcsAuth.AnonymousAccess, anonymousAccess) - overrideWithFlag(c, "kernel-list-cache-ttl-secs", &resolvedConfig.FileSystem.KernelListCacheTtlSecs, kernelListCacheTTLSecs) - overrideWithFlag(c, "max-retry-attempts", &resolvedConfig.GcsRetries.MaxRetryAttempts, maxRetryAttempts) - overrideWithFlag(c, "prometheus-port", &resolvedConfig.Metrics.PrometheusPort, prometheusPort) - - if err := cfg.ValidateConfig(&isSet{resolvedConfig}, resolvedConfig); err != nil { - return nil, fmt.Errorf("cfg.ValidateConfig: %w", err) - } - if err := cfg.Rationalize(&isSet{resolvedConfig}, resolvedConfig); err != nil { - return nil, fmt.Errorf("cfg.Rationalize: %w", err) - } - return resolvedConfig, nil -} - -// overrideWithFlag function overrides the toUpdate value with updateValue if -// the flag is set in cliContext. -func overrideWithFlag[T any](c cliContext, flag string, toUpdate *T, updateValue T) { - if !c.IsSet(flag) { - return - } - *toUpdate = updateValue -} - -// isSet is used to check if configs are explicitly set in the deprecated -// config file. -type isSet struct { - config *cfg.Config -} - -func (i *isSet) IsSet(flag string) bool { - switch flag { - case cfg.MetadataCacheTTLConfigKey: - return i.config.MetadataCache.TtlSecs != cfg.TtlInSecsUnsetSentinel - case cfg.StatCacheMaxSizeConfigKey: - return i.config.MetadataCache.StatCacheMaxSizeMb != cfg.StatCacheMaxSizeMBUnsetSentinel - } - return false -} diff --git a/cmd/legacy_param_mapper_test.go b/cmd/legacy_param_mapper_test.go deleted file mode 100644 index 02f8b3ba9f..0000000000 --- a/cmd/legacy_param_mapper_test.go +++ /dev/null @@ -1,658 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "fmt" - "net/url" - "os" - "path" - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" - mountpkg "github.com/googlecloudplatform/gcsfuse/v2/internal/mount" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/urfave/cli" -) - -type mockCLIContext struct { - cli.Context - isFlagSet map[string]bool -} - -func (m *mockCLIContext) IsSet(name string) bool { - return m.isFlagSet[name] -} - -func TestPopulateConfigFromLegacyFlags(t *testing.T) { - var populateConfigFromLegacyFlags = []struct { - testName string - legacyFlagStorage *flagStorage - mockCLICtx *mockCLIContext - legacyMountConfig *config.MountConfig - expectedConfig *cfg.Config - expectedErr error - }{ - { - testName: "nil flags", - legacyFlagStorage: nil, - mockCLICtx: &mockCLIContext{isFlagSet: map[string]bool{}}, - legacyMountConfig: &config.MountConfig{}, - expectedConfig: nil, - expectedErr: fmt.Errorf("PopulateNewConfigFromLegacyFlagsAndConfig: unexpected nil flags or mount config"), - }, - { - testName: "nil config", - legacyFlagStorage: &flagStorage{}, - mockCLICtx: &mockCLIContext{isFlagSet: map[string]bool{}}, - legacyMountConfig: nil, - expectedConfig: nil, - expectedErr: fmt.Errorf("PopulateNewConfigFromLegacyFlagsAndConfig: unexpected nil flags or mount config"), - }, - { - testName: "Test decode legacy flags.", - legacyFlagStorage: &flagStorage{ - AppName: "vertex", - Foreground: false, - ConfigFile: "~/Documents/config.yaml", - DirMode: 0755, - FileMode: 0711, - Uid: -1, - Gid: 17, - ImplicitDirs: true, - OnlyDir: "abc", - RenameDirLimit: 10, - IgnoreInterrupts: false, - CustomEndpoint: nil, - BillingProject: "billing-project", - KeyFile: "~/Documents/key-file", - TokenUrl: "tokenUrl", - ReuseTokenFromUrl: true, - EgressBandwidthLimitBytesPerSecond: 100, - OpRateLimitHz: 50, - SequentialReadSizeMb: 40, - AnonymousAccess: false, - MaxRetrySleep: 10, - MaxRetryAttempts: 100, - RetryMultiplier: 2, - StatCacheCapacity: 200, - StatCacheTTL: 50, - TypeCacheTTL: 70, - KernelListCacheTtlSeconds: 30, - HttpClientTimeout: 100, - TempDir: "~/temp", - MaxConnsPerHost: 200, - MaxIdleConnsPerHost: 150, - EnableNonexistentTypeCache: false, - StackdriverExportInterval: 40, - OtelCollectorAddress: "address", - PrometheusPort: 8080, - LogFile: "/tmp/log-file.json", - LogFormat: "json", - ExperimentalEnableJsonRead: true, - DebugGCS: true, - DebugFuse: true, - DebugInvariants: true, - DebugMutex: true, - ExperimentalMetadataPrefetchOnMount: "sync", - ClientProtocol: cfg.HTTP1, - }, - mockCLICtx: &mockCLIContext{ - isFlagSet: map[string]bool{ - "log-file": true, - "log-format": true, - "kernel-list-cache-ttl-secs": true, - "max-retry-attempts": true, - "prometheus-port": true, - }, - }, - legacyMountConfig: &config.MountConfig{ - FileCacheConfig: config.FileCacheConfig{ - CacheFileForRangeRead: false, - ParallelDownloadsPerFile: 16, - EnableCRC: false, - EnableParallelDownloads: false, - MaxParallelDownloads: 5, - MaxSizeMB: -1, - DownloadChunkSizeMB: 50, - }, - LogConfig: config.LogConfig{ - Severity: "INFO", - LogRotateConfig: config.LogRotateConfig{ - MaxFileSizeMB: 10, - BackupFileCount: 0, - Compress: false, - }, - }, - }, - expectedConfig: &cfg.Config{ - AppName: "vertex", - Foreground: false, - FileSystem: cfg.FileSystemConfig{ - DirMode: 493, // Octal(755) converted to decimal - FileMode: 457, // Octal(711) converted to decimal - Uid: -1, - Gid: 17, - RenameDirLimit: 10, - IgnoreInterrupts: false, - DisableParallelDirops: false, - FuseOptions: []string(nil), - TempDir: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "/temp")), - KernelListCacheTtlSecs: 30, - }, - ImplicitDirs: true, - OnlyDir: "abc", - CacheDir: "", - FileCache: cfg.FileCacheConfig{ - CacheFileForRangeRead: false, - ParallelDownloadsPerFile: 16, - EnableCrc: false, - EnableParallelDownloads: false, - MaxParallelDownloads: 5, - MaxSizeMb: -1, - DownloadChunkSizeMb: 50, - }, - GcsAuth: cfg.GcsAuthConfig{ - KeyFile: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "Documents/key-file")), - TokenUrl: "tokenUrl", - ReuseTokenFromUrl: true, - AnonymousAccess: false, - }, - GcsConnection: cfg.GcsConnectionConfig{ - CustomEndpoint: "", - BillingProject: "billing-project", - ClientProtocol: cfg.Protocol("http1"), - LimitBytesPerSec: 100, - LimitOpsPerSec: 50, - SequentialReadSizeMb: 40, - MaxConnsPerHost: 200, - MaxIdleConnsPerHost: 150, - HttpClientTimeout: 100, - ExperimentalEnableJsonRead: true, - }, - GcsRetries: cfg.GcsRetriesConfig{ - MaxRetrySleep: 10, - MaxRetryAttempts: 100, - Multiplier: 2, - }, - Logging: cfg.LoggingConfig{ - FilePath: cfg.ResolvedPath("/tmp/log-file.json"), - Severity: "TRACE", // Because debug fuse flag is set. - Format: "json", - LogRotate: cfg.LogRotateLoggingConfig{ - BackupFileCount: 0, - Compress: false, - MaxFileSizeMb: 10, - }, - }, - MetadataCache: cfg.MetadataCacheConfig{ - DeprecatedStatCacheCapacity: 200, - DeprecatedStatCacheTtl: 50, - DeprecatedTypeCacheTtl: 70, - EnableNonexistentTypeCache: false, - ExperimentalMetadataPrefetchOnMount: "sync", - }, - Metrics: cfg.MetricsConfig{ - StackdriverExportInterval: 40, - PrometheusPort: 8080, - }, - Monitoring: cfg.MonitoringConfig{ - ExperimentalOpentelemetryCollectorAddress: "address", - }, - Debug: cfg.DebugConfig{ - ExitOnInvariantViolation: true, - Gcs: true, - LogMutex: true, - Fuse: true, - }, - }, - expectedErr: nil, - }, - { - testName: "Test decode legacy config.", - legacyFlagStorage: &flagStorage{ - ClientProtocol: cfg.GRPC, - ExperimentalMetadataPrefetchOnMount: "disabled", - SequentialReadSizeMb: 200, - }, - mockCLICtx: &mockCLIContext{isFlagSet: map[string]bool{}}, - legacyMountConfig: &config.MountConfig{ - WriteConfig: cfg.WriteConfig{ - CreateEmptyFile: true, - }, - LogConfig: config.LogConfig{ - Severity: "info", - Format: "text", - FilePath: "~/Documents/log-file.txt", - LogRotateConfig: config.LogRotateConfig{ - MaxFileSizeMB: 20, - BackupFileCount: 2, - Compress: true, - }, - }, - FileCacheConfig: config.FileCacheConfig{ - MaxSizeMB: 20, - CacheFileForRangeRead: true, - EnableParallelDownloads: true, - ParallelDownloadsPerFile: 3, - MaxParallelDownloads: 6, - DownloadChunkSizeMB: 9, - EnableCRC: true, - WriteBufferSize: 1024 * 1024, - }, - CacheDir: "~/cache-dir", - MetadataCacheConfig: config.MetadataCacheConfig{ - TtlInSeconds: 200, - TypeCacheMaxSizeMB: 7, - StatCacheMaxSizeMB: 4, - }, - ListConfig: config.ListConfig{ - EnableEmptyManagedFolders: true, - }, - GCSConnection: config.GCSConnection{GRPCConnPoolSize: 29}, - GCSAuth: config.GCSAuth{AnonymousAccess: true}, - EnableHNS: true, - FileSystemConfig: config.FileSystemConfig{ - IgnoreInterrupts: true, - DisableParallelDirops: true, - KernelListCacheTtlSeconds: 30, - }, - }, - expectedConfig: &cfg.Config{ - Write: cfg.WriteConfig{CreateEmptyFile: true}, - Logging: cfg.LoggingConfig{ - Severity: "INFO", - Format: "text", - FilePath: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "Documents/log-file.txt")), - LogRotate: cfg.LogRotateLoggingConfig{ - MaxFileSizeMb: 20, - BackupFileCount: 2, - Compress: true, - }, - }, - FileCache: cfg.FileCacheConfig{ - MaxSizeMb: 20, - CacheFileForRangeRead: true, - EnableParallelDownloads: true, - ParallelDownloadsPerFile: 3, - MaxParallelDownloads: 6, - DownloadChunkSizeMb: 9, - EnableCrc: true, - WriteBufferSize: 1024 * 1024, - }, - CacheDir: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "cache-dir")), - MetadataCache: cfg.MetadataCacheConfig{ - TtlSecs: 200, - TypeCacheMaxSizeMb: 7, - StatCacheMaxSizeMb: 4, - ExperimentalMetadataPrefetchOnMount: "disabled", - }, - List: cfg.ListConfig{ - EnableEmptyManagedFolders: true, - }, - GcsConnection: cfg.GcsConnectionConfig{ - GrpcConnPoolSize: 29, - ClientProtocol: cfg.Protocol("grpc"), - SequentialReadSizeMb: 200, - }, - GcsAuth: cfg.GcsAuthConfig{AnonymousAccess: true}, - EnableHns: true, - FileSystem: cfg.FileSystemConfig{ - DisableParallelDirops: true, - IgnoreInterrupts: true, - KernelListCacheTtlSecs: 30, - }, - }, - expectedErr: nil, - }, - { - testName: "Test overlapping flags and configs set.", - legacyFlagStorage: &flagStorage{ - LogFile: "~/Documents/log-flag.txt", - LogFormat: "json", - IgnoreInterrupts: false, - AnonymousAccess: false, - KernelListCacheTtlSeconds: -1, - MaxRetryAttempts: 100, - ClientProtocol: cfg.HTTP2, - ExperimentalMetadataPrefetchOnMount: "disabled", - SequentialReadSizeMb: 200, - }, - mockCLICtx: &mockCLIContext{ - isFlagSet: map[string]bool{ - "log-file": true, - "log-format": true, - "ignore-interrupts": true, - "anonymous-access": true, - "kernel-list-cache-ttl-secs": true, - "max-retry-attempts": true, - }, - }, - legacyMountConfig: &config.MountConfig{ - FileCacheConfig: config.FileCacheConfig{ - CacheFileForRangeRead: false, - ParallelDownloadsPerFile: 16, - EnableCRC: false, - EnableParallelDownloads: false, - MaxParallelDownloads: 5, - MaxSizeMB: -1, - DownloadChunkSizeMB: 50, - }, - LogConfig: config.LogConfig{ - FilePath: "~/Documents/log-config.txt", - Format: "text", - Severity: "INFO", - LogRotateConfig: config.LogRotateConfig{ - MaxFileSizeMB: 1, - BackupFileCount: 0, - Compress: true, - }, - }, - FileSystemConfig: config.FileSystemConfig{ - IgnoreInterrupts: true, - KernelListCacheTtlSeconds: 100, - }, - GCSAuth: config.GCSAuth{ - AnonymousAccess: true, - }, - GCSRetries: config.GCSRetries{ - MaxRetryAttempts: 15, - }, - }, - expectedConfig: &cfg.Config{ - Logging: cfg.LoggingConfig{ - FilePath: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "/Documents/log-flag.txt")), - Format: "json", - Severity: "INFO", - LogRotate: cfg.LogRotateLoggingConfig{ - BackupFileCount: 0, - Compress: true, - MaxFileSizeMb: 1, - }, - }, - FileCache: cfg.FileCacheConfig{ - CacheFileForRangeRead: false, - ParallelDownloadsPerFile: 16, - EnableCrc: false, - EnableParallelDownloads: false, - MaxParallelDownloads: 5, - MaxSizeMb: -1, - DownloadChunkSizeMb: 50, - }, - FileSystem: cfg.FileSystemConfig{ - IgnoreInterrupts: false, - KernelListCacheTtlSecs: -1, - }, - GcsAuth: cfg.GcsAuthConfig{ - AnonymousAccess: false, - }, - MetadataCache: cfg.MetadataCacheConfig{ - ExperimentalMetadataPrefetchOnMount: "disabled", - }, - GcsConnection: cfg.GcsConnectionConfig{ - ClientProtocol: cfg.Protocol("http2"), - SequentialReadSizeMb: 200, - }, - GcsRetries: cfg.GcsRetriesConfig{ - MaxRetryAttempts: 100, - }, - }, - expectedErr: nil, - }, - } - - for _, tc := range populateConfigFromLegacyFlags { - t.Run(tc.testName, func(t *testing.T) { - resolvedConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(tc.mockCLICtx, tc.legacyFlagStorage, tc.legacyMountConfig) - - if assert.Equal(t, tc.expectedErr, err) { - assert.Equal(t, tc.expectedConfig, resolvedConfig) - } - }) - } -} - -func TestPopulateConfigFromLegacyFlags_KeyFileResolution(t *testing.T) { - currentWorkingDir, err := os.Getwd() - require.Nil(t, err) - var keyFileTests = []struct { - testName string - givenKeyFile string - expectedKeyFile cfg.ResolvedPath - }{ - { - testName: "absolute path", - givenKeyFile: "/tmp/key-file.json", - expectedKeyFile: "/tmp/key-file.json", - }, - { - testName: "relative path", - givenKeyFile: "~/Documents/key-file.json", - expectedKeyFile: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "/Documents/key-file.json")), - }, - { - testName: "current working directory", - givenKeyFile: "key-file.json", - expectedKeyFile: cfg.ResolvedPath(path.Join(currentWorkingDir, "key-file.json")), - }, - { - testName: "empty path", - givenKeyFile: "", - expectedKeyFile: "", - }, - } - - for _, tc := range keyFileTests { - t.Run(tc.testName, func(t *testing.T) { - mockCLICtx := &mockCLIContext{} - legacyFlagStorage := &flagStorage{ - ClientProtocol: cfg.HTTP2, - KeyFile: tc.givenKeyFile, - } - legacyMountCfg := &config.MountConfig{} - - resolvedConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(mockCLICtx, legacyFlagStorage, legacyMountCfg) - - if assert.Nil(t, err) { - assert.Equal(t, tc.expectedKeyFile, resolvedConfig.GcsAuth.KeyFile) - } - }) - } -} - -func TestPopulateConfigFromLegacyFlags_LogFileResolution(t *testing.T) { - currentWorkingDir, err := os.Getwd() - require.Nil(t, err) - var logFileTests = []struct { - testName string - givenLogFile string - expectedLogFile cfg.ResolvedPath - }{ - { - testName: "absolute path", - givenLogFile: "/tmp/log-file.json", - expectedLogFile: "/tmp/log-file.json", - }, - { - testName: "relative path", - givenLogFile: "~/Documents/log-file.json", - expectedLogFile: cfg.ResolvedPath(path.Join(os.Getenv("HOME"), "Documents/log-file.json")), - }, - { - testName: "current working directory", - givenLogFile: "log-file.json", - expectedLogFile: cfg.ResolvedPath(path.Join(currentWorkingDir, "log-file.json")), - }, - { - testName: "empty path", - givenLogFile: "", - expectedLogFile: "", - }, - } - - for _, tc := range logFileTests { - t.Run(tc.testName, func(t *testing.T) { - mockCLICtx := &mockCLIContext{} - legacyFlagStorage := &flagStorage{ - ClientProtocol: cfg.HTTP2, - LogFile: tc.givenLogFile, - } - legacyMountCfg := &config.MountConfig{} - - resolvedConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(mockCLICtx, legacyFlagStorage, legacyMountCfg) - - if assert.Nil(t, err) { - assert.Equal(t, tc.expectedLogFile, resolvedConfig.Logging.FilePath) - } - }) - } -} - -func TestCustomEndpointResolutionFromFlags(t *testing.T) { - u, err := url.Parse("http://abc.xyz") - require.Nil(t, err) - legacyFlagStorage := &flagStorage{ - ClientProtocol: cfg.HTTP2, - CustomEndpoint: u, - } - - resolvedConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, legacyFlagStorage, &config.MountConfig{}) - - if assert.Nil(t, err) && assert.NotEmpty(t, resolvedConfig.GcsConnection.CustomEndpoint) { - assert.Equal(t, resolvedConfig.GcsConnection.CustomEndpoint, u.String()) - } -} - -func TestConfigValidation(t *testing.T) { - conf := config.NewMountConfig() - conf.LogRotateConfig.MaxFileSizeMB = -1 - - _, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, &flagStorage{ClientProtocol: mountpkg.ClientProtocol(cfg.HTTP2)}, conf) - - assert.Error(t, err) -} - -func TestValidClientProtocol(t *testing.T) { - flags := &flagStorage{ - ClientProtocol: mountpkg.ClientProtocol("http1"), - } - - v, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, flags, &config.MountConfig{}) - - if assert.Nil(t, err) { - assert.Equal(t, v.GcsConnection.ClientProtocol, cfg.Protocol("http1")) - } -} - -func TestInvalidClientProtocol(t *testing.T) { - flags := &flagStorage{ - ClientProtocol: mountpkg.ClientProtocol("http3"), - } - - _, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, flags, &config.MountConfig{}) - - assert.NotNil(t, err) -} - -func TestLogSeverityRationalization(t *testing.T) { - testCases := []struct { - name string - cfgSev string - debugFuse bool - debugGCS bool - debugMutex bool - expected cfg.LogSeverity - }{ - { - name: "debugFuse set to true", - cfgSev: "INFO", - debugFuse: true, - debugGCS: false, - debugMutex: false, - expected: "TRACE", - }, - { - name: "debugGCS set to true", - cfgSev: "ERROR", - debugFuse: false, - debugGCS: true, - debugMutex: false, - expected: "TRACE", - }, - { - name: "debugMutex set to true", - cfgSev: "WARNING", - debugFuse: false, - debugGCS: false, - debugMutex: true, - expected: "TRACE", - }, - { - name: "multiple debug flags set to true", - cfgSev: "INFO", - debugFuse: true, - debugGCS: false, - debugMutex: true, - expected: "TRACE", - }, - { - name: "no debug flags set to true", - cfgSev: "INFO", - debugFuse: false, - debugGCS: false, - debugMutex: false, - expected: "INFO", - }, - } - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - flags := &flagStorage{ - ClientProtocol: mountpkg.ClientProtocol("http1"), - ExperimentalMetadataPrefetchOnMount: "disabled", - SequentialReadSizeMb: 200, - DebugFuse: tc.debugFuse, - DebugGCS: tc.debugGCS, - DebugMutex: tc.debugMutex, - } - c := config.NewMountConfig() - c.Severity = tc.cfgSev - - resolvedConfig, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{isFlagSet: map[string]bool{ - "debug_fuse": true, - "debug_gcs": true, - "debug_mutex": true, - }}, flags, c) - - if assert.NoError(t, err) { - assert.Equal(t, tc.expected, resolvedConfig.Logging.Severity) - } - }) - } -} - -func TestPopulateConfigFromLegacyFlags_MountOption(t *testing.T) { - flags := &flagStorage{ - MountOptions: []string{"rw,nodev", "user=jacobsa,noauto"}, - ClientProtocol: mountpkg.ClientProtocol(cfg.HTTP2), - } - - v, err := PopulateNewConfigFromLegacyFlagsAndConfig(&mockCLIContext{}, flags, &config.MountConfig{}) - - if assert.Nil(t, err) { - assert.Equal(t, v.FileSystem.FuseOptions, []string{"rw,nodev", "user=jacobsa,noauto"}) - } -} diff --git a/cmd/root.go b/cmd/root.go index 81bcadbbb1..105a6cf4a4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -104,11 +104,11 @@ of Cloud Storage FUSE, see https://cloud.google.com/storage/docs/gcs-fuse.`, return rootCmd, nil } -// ConvertToPosixArgs converts a slice of commandline args and transforms them +// convertToPosixArgs converts a slice of commandline args and transforms them // into POSIX compliant args. All it does is that it converts flags specified // using a single-hyphen to double-hyphens. We are excluding "-v" because it's // reserved for showing version in Cobra. -func ConvertToPosixArgs(args []string, c *cobra.Command) []string { +func convertToPosixArgs(args []string, c *cobra.Command) []string { pArgs := make([]string, 0, len(args)) flagSet := make(map[string]bool) c.PersistentFlags().VisitAll(func(f *pflag.Flag) { @@ -146,12 +146,12 @@ func ConvertToPosixArgs(args []string, c *cobra.Command) []string { return pArgs } -var ExecuteNewMain = func() { +var ExecuteMountCmd = func() { rootCmd, err := NewRootCmd(Mount) if err != nil { log.Fatalf("Error occurred while creating the root command: %v", err) } - rootCmd.SetArgs(ConvertToPosixArgs(os.Args, rootCmd)) + rootCmd.SetArgs(convertToPosixArgs(os.Args, rootCmd)) if err := rootCmd.Execute(); err != nil { log.Fatalf("Error occurred during command execution: %v", err) } diff --git a/cmd/root_test.go b/cmd/root_test.go index 45b1be135e..54b6c4ffb3 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -34,7 +34,7 @@ func TestDefaultMaxParallelDownloads(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs([]string{"abc", "pqr"}, cmd)) + cmd.SetArgs(convertToPosixArgs([]string{"abc", "pqr"}, cmd)) if assert.Nil(t, cmd.Execute()) { assert.LessOrEqual(t, int64(16), actual.FileCache.MaxParallelDownloads) @@ -73,7 +73,7 @@ func TestCobraArgsNumInRange(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cmd, err := NewRootCmd(func(*cfg.Config, string, string) error { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -132,7 +132,7 @@ func TestArgsParsing_MountPoint(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -180,7 +180,7 @@ func TestArgsParsing_MountOptions(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -283,7 +283,7 @@ func TestArgsParsing_WriteConfigFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -347,7 +347,7 @@ func TestArgsParsing_FileCacheFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -399,7 +399,7 @@ func TestArgParsing_ExperimentalMetadataPrefetchFlag(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -431,7 +431,7 @@ func TestArgParsing_ExperimentalMetadataPrefetchFlag_Failed(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -482,7 +482,7 @@ func TestArgsParsing_GCSAuthFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -519,7 +519,7 @@ func TestArgsParsing_GCSAuthFlagsThrowsError(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) assert.Error(t, cmd.Execute()) }) @@ -580,7 +580,7 @@ func TestArgsParsing_GCSConnectionFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -623,7 +623,7 @@ func TestArgsParsing_GCSConnectionFlagsThrowsError(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) assert.Error(t, cmd.Execute()) }) @@ -702,7 +702,7 @@ func TestArgsParsing_FileSystemFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -746,7 +746,7 @@ func TestArgsParsing_FileSystemFlagsThrowsError(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) assert.Error(t, cmd.Execute()) }) @@ -783,7 +783,7 @@ func TestArgsParsing_ListFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -820,7 +820,7 @@ func TestArgsParsing_EnableHNSFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() @@ -879,7 +879,7 @@ func TestArgsParsing_MetadataCacheFlags(t *testing.T) { return nil }) require.Nil(t, err) - cmd.SetArgs(ConvertToPosixArgs(tc.args, cmd)) + cmd.SetArgs(convertToPosixArgs(tc.args, cmd)) err = cmd.Execute() diff --git a/go.mod b/go.mod index 8eaf6a293b..27261f3d5c 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 - github.com/urfave/cli v1.22.16 go.opencensus.io v0.24.0 go.opentelemetry.io/contrib/detectors/gcp v1.31.0 go.opentelemetry.io/otel v1.31.0 @@ -65,7 +64,6 @@ require ( github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/envoyproxy/go-control-plane v0.13.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect @@ -99,7 +97,6 @@ require ( github.com/prometheus/prometheus v0.35.0 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect diff --git a/go.sum b/go.sum index fb13f6a1d1..9abb440ad3 100644 --- a/go.sum +++ b/go.sum @@ -110,7 +110,6 @@ github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.2 h1:cZpsGsWTIFKymTA0je7IIvi1O7Es7apb9CF3EQlOcfE= @@ -354,8 +353,6 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= -github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -1062,7 +1059,6 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1168,8 +1164,6 @@ github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= -github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= diff --git a/internal/config/mount_config.go b/internal/config/mount_config.go deleted file mode 100644 index 6554f95355..0000000000 --- a/internal/config/mount_config.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/util" -) - -const ( - DefaultEnableEmptyManagedFoldersListing = false - DefaultGrpcConnPoolSize = 1 - DefaultAnonymousAccess = false - DefaultEnableHNS = true - DefaultIgnoreInterrupts = true - DefaultPrometheusPort = 0 - - DefaultKernelListCacheTtlSeconds int64 = 0 - DefaultMaxRetryAttempts = int64(0) - - // File Cache Config constants. - - DefaultFileCacheMaxSizeMB = -1 - DefaultEnableCRC = false - DefaultEnableParallelDownloads = false - DefaultDownloadChunkSizeMB = 50 - DefaultParallelDownloadsPerFile = 16 - DefaultWriteBufferSize = int64(4 * util.MiB) - DefaultEnableODirect = false - - // DefaultTypeCacheMaxSizeMB is the default value of type-cache max-size for every directory in MiBs. - // The value is set at the size needed for about 21k type-cache entries, - // each of which is about 200 bytes in size. - DefaultTypeCacheMaxSizeMB = 4 -) - -type LogConfig struct { - Severity string `yaml:"severity"` - Format string `yaml:"format"` - FilePath string `yaml:"file-path"` - LogRotateConfig LogRotateConfig `yaml:"log-rotate"` -} - -type MetricsConfig struct { - // Expose Prometheus metrics endpoint on this port and a path of /metrics. - PrometheusPort int `yaml:"prometheus-port"` -} - -type ListConfig struct { - // This flag is specially added to handle the corner case in listing managed folders. - // There are two corner cases (a) empty managed folder (b) nested managed folder which doesn't contain any descendent as object. - // This flag always works in conjunction with ImplicitDirectories flag. - // - // (a) If only ImplicitDirectories is true, all managed folders are listed other than above two mentioned cases. - // (b) If both ImplicitDirectories and EnableEmptyManagedFolders are true, then all the managed folders are listed including the above-mentioned corner case. - // (c) If ImplicitDirectories is false then no managed folders are listed irrespective of EnableEmptyManagedFolders flag. - EnableEmptyManagedFolders bool `yaml:"enable-empty-managed-folders"` -} - -type GCSConnection struct { - // GRPCConnPoolSize configures the number of gRPC channel in grpc client. - GRPCConnPoolSize int `yaml:"grpc-conn-pool-size,omitempty"` -} - -type GCSAuth struct { - // Authentication is enabled by default. The skip flag disables authentication. For users of the --custom-endpoint flag, - // please pass anonymous-access flag explicitly if you do not want authentication enabled for your workflow. - AnonymousAccess bool `yaml:"anonymous-access"` -} - -type FileSystemConfig struct { - IgnoreInterrupts bool `yaml:"ignore-interrupts"` - DisableParallelDirops bool `yaml:"disable-parallel-dirops"` - KernelListCacheTtlSeconds int64 `yaml:"kernel-list-cache-ttl-secs"` -} - -type FileCacheConfig struct { - MaxSizeMB int64 `yaml:"max-size-mb"` - CacheFileForRangeRead bool `yaml:"cache-file-for-range-read"` - EnableParallelDownloads bool `yaml:"enable-parallel-downloads,omitempty"` - ParallelDownloadsPerFile int `yaml:"parallel-downloads-per-file,omitempty"` - MaxParallelDownloads int `yaml:"max-parallel-downloads,omitempty"` - DownloadChunkSizeMB int `yaml:"download-chunk-size-mb,omitempty"` - EnableCRC bool `yaml:"enable-crc"` - WriteBufferSize int64 `yaml:"write-buffer-size,omitempty"` - EnableODirect bool `yaml:"enable-o-direct"` -} - -type MetadataCacheConfig struct { - // TtlInSeconds is the ttl - // value in seconds, to be used for stat-cache and type-cache. - // It can be set to -1 for no-ttl, 0 for - // no cache and > 0 for ttl-controlled metadata-cache. - // Any value set below -1 will throw an error. - TtlInSeconds int64 `yaml:"ttl-secs,omitempty"` - // TypeCacheMaxSizeMB is the upper limit - // on the maximum size of type-cache maps, - // which are currently - // maintained at per-directory level. - TypeCacheMaxSizeMB int `yaml:"type-cache-max-size-mb,omitempty"` - - // StatCacheMaxSizeMB is the maximum size of stat-cache - // in MiBs. - // It can also be set to -1 for no-size-limit, 0 for - // no cache. Values below -1 are not supported. - StatCacheMaxSizeMB int64 `yaml:"stat-cache-max-size-mb,omitempty"` -} - -type GCSRetries struct { - // Set max retry attempts in case of retryable errors. Default value is 6. - MaxRetryAttempts int64 `yaml:"max-retry-attempts"` -} - -type MountConfig struct { - cfg.WriteConfig `yaml:"write"` - LogConfig `yaml:"logging"` - FileCacheConfig `yaml:"file-cache"` - CacheDir string `yaml:"cache-dir"` - MetadataCacheConfig `yaml:"metadata-cache"` - ListConfig `yaml:"list"` - GCSConnection `yaml:"gcs-connection"` - GCSAuth `yaml:"gcs-auth"` - EnableHNS bool `yaml:"enable-hns"` - FileSystemConfig `yaml:"file-system"` - GCSRetries `yaml:"gcs-retries"` - MetricsConfig `yaml:"metrics"` -} - -// LogRotateConfig defines the parameters for log rotation. It consists of three -// configuration options: -// 1. max-file-size-mb: specifies the maximum size in megabytes that a log file -// can reach before it is rotated. The default value is 512 megabytes. -// 2. backup-file-count: determines the maximum number of backup log files to -// retain after they have been rotated. The default value is 10. When value is -// set to 0, all backup files are retained. -// 3. compress: indicates whether the rotated log files should be compressed -// using gzip. The default value is False. -type LogRotateConfig struct { - MaxFileSizeMB int `yaml:"max-file-size-mb"` - BackupFileCount int `yaml:"backup-file-count"` - Compress bool `yaml:"compress"` -} - -func NewMountConfig() *MountConfig { - mountConfig := &MountConfig{} - logConfig := cfg.DefaultLoggingConfig() - logRotateConfig := logConfig.LogRotate - mountConfig.LogConfig = LogConfig{ - // Making the default severity as INFO. - Severity: string(logConfig.Severity), - // Setting default values of log rotate config. - LogRotateConfig: LogRotateConfig{ - MaxFileSizeMB: int(logRotateConfig.MaxFileSizeMb), - BackupFileCount: int(logRotateConfig.BackupFileCount), - Compress: logRotateConfig.Compress, - }, - } - mountConfig.FileCacheConfig = FileCacheConfig{ - MaxSizeMB: DefaultFileCacheMaxSizeMB, - EnableParallelDownloads: DefaultEnableParallelDownloads, - ParallelDownloadsPerFile: DefaultParallelDownloadsPerFile, - MaxParallelDownloads: cfg.DefaultMaxParallelDownloads(), - DownloadChunkSizeMB: DefaultDownloadChunkSizeMB, - EnableCRC: DefaultEnableCRC, - WriteBufferSize: DefaultWriteBufferSize, - EnableODirect: DefaultEnableODirect, - } - mountConfig.MetadataCacheConfig = MetadataCacheConfig{ - TtlInSeconds: cfg.TtlInSecsUnsetSentinel, - TypeCacheMaxSizeMB: DefaultTypeCacheMaxSizeMB, - StatCacheMaxSizeMB: cfg.StatCacheMaxSizeMBUnsetSentinel, - } - mountConfig.ListConfig = ListConfig{ - EnableEmptyManagedFolders: DefaultEnableEmptyManagedFoldersListing, - } - mountConfig.GCSConnection = GCSConnection{ - GRPCConnPoolSize: DefaultGrpcConnPoolSize, - } - mountConfig.GCSAuth = GCSAuth{ - AnonymousAccess: DefaultAnonymousAccess, - } - mountConfig.EnableHNS = DefaultEnableHNS - - mountConfig.FileSystemConfig = FileSystemConfig{ - KernelListCacheTtlSeconds: DefaultKernelListCacheTtlSeconds, - } - - mountConfig.FileSystemConfig.IgnoreInterrupts = DefaultIgnoreInterrupts - - mountConfig.GCSRetries = GCSRetries{ - MaxRetryAttempts: DefaultMaxRetryAttempts, - } - - mountConfig.MetricsConfig = MetricsConfig{ - PrometheusPort: DefaultPrometheusPort, - } - - return mountConfig -} diff --git a/internal/config/testdata/empty_file.yaml b/internal/config/testdata/empty_file.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/internal/config/testdata/file_system_config/invalid_disable_parallel_dirops.yaml b/internal/config/testdata/file_system_config/invalid_disable_parallel_dirops.yaml deleted file mode 100644 index 1109956504..0000000000 --- a/internal/config/testdata/file_system_config/invalid_disable_parallel_dirops.yaml +++ /dev/null @@ -1,2 +0,0 @@ -file-system: - disable-parallel-dirops: -1 diff --git a/internal/config/testdata/file_system_config/invalid_ignore_interrupts.yaml b/internal/config/testdata/file_system_config/invalid_ignore_interrupts.yaml deleted file mode 100644 index a381313bf1..0000000000 --- a/internal/config/testdata/file_system_config/invalid_ignore_interrupts.yaml +++ /dev/null @@ -1,2 +0,0 @@ -file-system: - ignore-interrupts: abc diff --git a/internal/config/testdata/file_system_config/unset_disable_parallel_dirops.yaml b/internal/config/testdata/file_system_config/unset_disable_parallel_dirops.yaml deleted file mode 100644 index 4db0c34ced..0000000000 --- a/internal/config/testdata/file_system_config/unset_disable_parallel_dirops.yaml +++ /dev/null @@ -1,2 +0,0 @@ -write: - create-empty-file: true diff --git a/internal/config/testdata/file_system_config/unset_ignore_interrupts.yaml b/internal/config/testdata/file_system_config/unset_ignore_interrupts.yaml deleted file mode 100644 index 4db0c34ced..0000000000 --- a/internal/config/testdata/file_system_config/unset_ignore_interrupts.yaml +++ /dev/null @@ -1,2 +0,0 @@ -write: - create-empty-file: true diff --git a/internal/config/testdata/file_system_config/unset_kernel_list_cache_ttl.yaml b/internal/config/testdata/file_system_config/unset_kernel_list_cache_ttl.yaml deleted file mode 100644 index 2b179a552a..0000000000 --- a/internal/config/testdata/file_system_config/unset_kernel_list_cache_ttl.yaml +++ /dev/null @@ -1 +0,0 @@ -file-system: diff --git a/internal/config/testdata/file_system_config/valid_kernel_list_cache_ttl.yaml b/internal/config/testdata/file_system_config/valid_kernel_list_cache_ttl.yaml deleted file mode 100644 index 79de4541c4..0000000000 --- a/internal/config/testdata/file_system_config/valid_kernel_list_cache_ttl.yaml +++ /dev/null @@ -1,2 +0,0 @@ -file-system: - kernel-list-cache-ttl-secs: 10 diff --git a/internal/config/testdata/gcs_auth/invalid_anonymous_access.yaml b/internal/config/testdata/gcs_auth/invalid_anonymous_access.yaml deleted file mode 100644 index bcd578dd3e..0000000000 --- a/internal/config/testdata/gcs_auth/invalid_anonymous_access.yaml +++ /dev/null @@ -1,2 +0,0 @@ -gcs-auth: - anonymous-access: abc diff --git a/internal/config/testdata/gcs_auth/unset_anonymous_access.yaml b/internal/config/testdata/gcs_auth/unset_anonymous_access.yaml deleted file mode 100644 index 4db0c34ced..0000000000 --- a/internal/config/testdata/gcs_auth/unset_anonymous_access.yaml +++ /dev/null @@ -1,2 +0,0 @@ -write: - create-empty-file: true diff --git a/internal/config/testdata/gcs_connection/invalid_conn_pool_size.yaml b/internal/config/testdata/gcs_connection/invalid_conn_pool_size.yaml deleted file mode 100644 index fc3178dacd..0000000000 --- a/internal/config/testdata/gcs_connection/invalid_conn_pool_size.yaml +++ /dev/null @@ -1,3 +0,0 @@ -gcs-connection: - grpc-conn-pool-size: 0 - diff --git a/internal/config/testdata/gcs_connection/unset_conn_pool_size.yaml b/internal/config/testdata/gcs_connection/unset_conn_pool_size.yaml deleted file mode 100644 index eb9dd99d94..0000000000 --- a/internal/config/testdata/gcs_connection/unset_conn_pool_size.yaml +++ /dev/null @@ -1,3 +0,0 @@ -write: - create-empty-file: true - diff --git a/internal/config/testdata/invalid_config.yaml b/internal/config/testdata/invalid_config.yaml deleted file mode 100644 index dfd3ae1d23..0000000000 --- a/internal/config/testdata/invalid_config.yaml +++ /dev/null @@ -1 +0,0 @@ -test= value diff --git a/internal/config/testdata/invalid_log_config.yaml b/internal/config/testdata/invalid_log_config.yaml deleted file mode 100644 index 335382b85b..0000000000 --- a/internal/config/testdata/invalid_log_config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -write: - create-empty-file: true -logging: - severity: critical diff --git a/internal/config/testdata/invalid_unexpectedfield_config.yaml b/internal/config/testdata/invalid_unexpectedfield_config.yaml deleted file mode 100644 index 9a32707f8d..0000000000 --- a/internal/config/testdata/invalid_unexpectedfield_config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -write: - create-empty-file: true -logging: - # The correct field is 'format'. - formats: text \ No newline at end of file diff --git a/internal/config/testdata/metadata_cache_config_stat-cache-max-size-mb_unset.yaml b/internal/config/testdata/metadata_cache_config_stat-cache-max-size-mb_unset.yaml deleted file mode 100644 index 80142e2af4..0000000000 --- a/internal/config/testdata/metadata_cache_config_stat-cache-max-size-mb_unset.yaml +++ /dev/null @@ -1,2 +0,0 @@ -metadata-cache: - diff --git a/internal/config/testdata/metadata_cache_config_ttl-unset.yaml b/internal/config/testdata/metadata_cache_config_ttl-unset.yaml deleted file mode 100644 index 0a5f5b0e60..0000000000 --- a/internal/config/testdata/metadata_cache_config_ttl-unset.yaml +++ /dev/null @@ -1 +0,0 @@ -metadata-cache: diff --git a/internal/config/testdata/metadata_cache_config_type-cache-max-size-mb_unset.yaml b/internal/config/testdata/metadata_cache_config_type-cache-max-size-mb_unset.yaml deleted file mode 100644 index 80142e2af4..0000000000 --- a/internal/config/testdata/metadata_cache_config_type-cache-max-size-mb_unset.yaml +++ /dev/null @@ -1,2 +0,0 @@ -metadata-cache: - diff --git a/internal/config/testdata/metrics_config/unset_prometheus_port.yaml b/internal/config/testdata/metrics_config/unset_prometheus_port.yaml deleted file mode 100644 index 4db0c34ced..0000000000 --- a/internal/config/testdata/metrics_config/unset_prometheus_port.yaml +++ /dev/null @@ -1,2 +0,0 @@ -write: - create-empty-file: true diff --git a/internal/config/testdata/valid_config.yaml b/internal/config/testdata/valid_config.yaml deleted file mode 100644 index 823dcae407..0000000000 --- a/internal/config/testdata/valid_config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -write: - create-empty-file: true -logging: - file-path: /tmp/logfile.json - format: text - severity: error - log-rotate: - max-file-size-mb: 100 - backup-file-count: 5 - compress: false -cache-dir: "/tmp/read_cache/" -file-cache: - max-size-mb: 100 - cache-file-for-range-read: true - enable-parallel-downloads: true - parallel-downloads-per-file: 10 - max-parallel-downloads: -1 - download-chunk-size-mb: 100 - enable-crc: false - write-buffer-size: 8192 - enable-o-direct: false -metadata-cache: - ttl-secs: 5 - type-cache-max-size-mb: 1 - stat-cache-max-size-mb: 3 -gcs-auth: - anonymous-access: true -list: - enable-empty-managed-folders: true -gcs-connection: - grpc-conn-pool-size: 4 -enable-hns: true -file-system: - ignore-interrupts: true - disable-parallel-dirops: true -gcs-retries: - max-retry-attempts: 6 -metrics: - prometheus-port: 8080 diff --git a/internal/config/yaml_parser.go b/internal/config/yaml_parser.go deleted file mode 100644 index 00e554ca2a..0000000000 --- a/internal/config/yaml_parser.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "bytes" - "fmt" - "io" - "os" - "strings" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "gopkg.in/yaml.v3" -) - -const ( - parseConfigFileErrMsgFormat = "error parsing config file: %v" -) - -func IsValidLogSeverity(severity string) bool { - switch severity { - case - cfg.TRACE, - cfg.DEBUG, - cfg.INFO, - cfg.WARNING, - cfg.ERROR, - cfg.OFF: - return true - } - return false -} - -func (grpcClientConfig *GCSConnection) validate() error { - if grpcClientConfig.GRPCConnPoolSize < 1 { - return fmt.Errorf("the value of conn-pool-size can't be less than 1") - } - return nil -} - -func ParseConfigFile(fileName string) (mountConfig *MountConfig, err error) { - mountConfig = NewMountConfig() - - if fileName == "" { - return - } - - buf, err := os.ReadFile(fileName) - if err != nil { - err = fmt.Errorf("error reading config file: %w", err) - return - } - - // Ensure error is thrown when unexpected configs are passed in config file. - // Ref: https://github.com/go-yaml/yaml/issues/602#issuecomment-623485602 - decoder := yaml.NewDecoder(bytes.NewReader(buf)) - decoder.KnownFields(true) - if err = decoder.Decode(mountConfig); err != nil { - // Decode returns EOF in case of empty config file. - if err == io.EOF { - return mountConfig, nil - } - return mountConfig, fmt.Errorf(parseConfigFileErrMsgFormat, err) - } - - // convert log severity to upper-case - mountConfig.LogConfig.Severity = strings.ToUpper(mountConfig.LogConfig.Severity) - if !IsValidLogSeverity(mountConfig.LogConfig.Severity) { - err = fmt.Errorf("error parsing config file: log severity should be one of [trace, debug, info, warning, error, off]") - return - } - - if err = mountConfig.GCSConnection.validate(); err != nil { - return mountConfig, fmt.Errorf("error parsing gcs-connection configs: %w", err) - } - - // The EnableEmptyManagedFolders flag must be set to true to enforce folder prefixes for Hierarchical buckets. - if mountConfig.EnableHNS { - mountConfig.ListConfig.EnableEmptyManagedFolders = true - } - - return -} diff --git a/internal/config/yaml_parser_test.go b/internal/config/yaml_parser_test.go deleted file mode 100644 index 4f2fe7c486..0000000000 --- a/internal/config/yaml_parser_test.go +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright 2021 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "fmt" - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/cfg" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" -) - -type YamlParserTest struct { - suite.Suite -} - -func TestYamlParserSuite(t *testing.T) { - suite.Run(t, new(YamlParserTest)) -} -func validateDefaultConfig(t *testing.T, mountConfig *MountConfig) { - assert.NotNil(t, mountConfig) - assert.False(t, mountConfig.CreateEmptyFile) - assert.False(t, mountConfig.ListConfig.EnableEmptyManagedFolders) - assert.Equal(t, "INFO", string(mountConfig.LogConfig.Severity)) - assert.Equal(t, "", mountConfig.LogConfig.Format) - assert.Equal(t, "", mountConfig.LogConfig.FilePath) - assert.Equal(t, 512, mountConfig.LogConfig.LogRotateConfig.MaxFileSizeMB) - assert.Equal(t, 10, mountConfig.LogConfig.LogRotateConfig.BackupFileCount) - assert.True(t, bool(mountConfig.LogConfig.LogRotateConfig.Compress)) - assert.Equal(t, "", string(mountConfig.CacheDir)) - assert.Equal(t, int64(-1), mountConfig.FileCacheConfig.MaxSizeMB) - assert.False(t, mountConfig.FileCacheConfig.CacheFileForRangeRead) - assert.False(t, mountConfig.FileCacheConfig.EnableParallelDownloads) - assert.Equal(t, 16, mountConfig.FileCacheConfig.ParallelDownloadsPerFile) - assert.GreaterOrEqual(t, mountConfig.FileCacheConfig.MaxParallelDownloads, 16) - assert.Equal(t, 50, mountConfig.FileCacheConfig.DownloadChunkSizeMB) - assert.False(t, mountConfig.FileCacheConfig.EnableCRC) - assert.Equal(t, int64(4*1024*1024), mountConfig.FileCacheConfig.WriteBufferSize) - assert.Equal(t, false, mountConfig.FileCacheConfig.EnableODirect) - assert.Equal(t, 1, mountConfig.GCSConnection.GRPCConnPoolSize) - assert.False(t, mountConfig.GCSAuth.AnonymousAccess) - assert.True(t, mountConfig.EnableHNS) - assert.True(t, mountConfig.FileSystemConfig.IgnoreInterrupts) - assert.False(t, mountConfig.FileSystemConfig.DisableParallelDirops) - assert.Equal(t, DefaultKernelListCacheTtlSeconds, mountConfig.KernelListCacheTtlSeconds) - assert.Equal(t, DefaultMaxRetryAttempts, mountConfig.GCSRetries.MaxRetryAttempts) -} - -func (t *YamlParserTest) TestReadConfigFile_EmptyFileName() { - mountConfig, err := ParseConfigFile("") - - assert.NoError(t.T(), err) - validateDefaultConfig(t.T(), mountConfig) -} - -func (t *YamlParserTest) TestReadConfigFile_EmptyFile() { - mountConfig, err := ParseConfigFile("testdata/empty_file.yaml") - - assert.NoError(t.T(), err) - validateDefaultConfig(t.T(), mountConfig) -} - -func (t *YamlParserTest) TestReadConfigFile_NonExistingFile() { - _, err := ParseConfigFile("testdata/nofile.yaml") - - assert.ErrorContains(t.T(), err, "error reading config file: open testdata/nofile.yaml: no such file or directory") -} - -func (t *YamlParserTest) TestReadConfigFile_InvalidConfig() { - _, err := ParseConfigFile("testdata/invalid_config.yaml") - - assert.ErrorContains(t.T(), err, "error parsing config file: yaml: unmarshal errors:") -} - -func (t *YamlParserTest) TestReadConfigFile_Invalid_UnexpectedField_Config() { - _, err := ParseConfigFile("testdata/invalid_unexpectedfield_config.yaml") - - assert.ErrorContains(t.T(), err, "error parsing config file: yaml: unmarshal errors:") - assert.ErrorContains(t.T(), err, "line 5: field formats not found in type config.LogConfig") -} - -func (t *YamlParserTest) TestReadConfigFile_ValidConfig() { - mountConfig, err := ParseConfigFile("testdata/valid_config.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.True(t.T(), mountConfig.WriteConfig.CreateEmptyFile) - assert.Equal(t.T(), cfg.ERROR, mountConfig.LogConfig.Severity) - assert.Equal(t.T(), "/tmp/logfile.json", mountConfig.LogConfig.FilePath) - assert.Equal(t.T(), "text", mountConfig.LogConfig.Format) - - // log-rotate config - assert.Equal(t.T(), 100, mountConfig.LogConfig.LogRotateConfig.MaxFileSizeMB) - assert.Equal(t.T(), 5, mountConfig.LogConfig.LogRotateConfig.BackupFileCount) - assert.False(t.T(), mountConfig.LogConfig.LogRotateConfig.Compress) - - // metadata-cache config - assert.Equal(t.T(), int64(5), mountConfig.MetadataCacheConfig.TtlInSeconds) - assert.Equal(t.T(), 1, mountConfig.MetadataCacheConfig.TypeCacheMaxSizeMB) - assert.Equal(t.T(), int64(3), mountConfig.MetadataCacheConfig.StatCacheMaxSizeMB) - - // list config - assert.True(t.T(), mountConfig.ListConfig.EnableEmptyManagedFolders) - - // auth config - assert.True(t.T(), mountConfig.GCSAuth.AnonymousAccess) - - // enable-hns - assert.True(t.T(), bool(mountConfig.EnableHNS)) - - // file-system config - assert.True(t.T(), mountConfig.FileSystemConfig.IgnoreInterrupts) - assert.True(t.T(), mountConfig.FileSystemConfig.DisableParallelDirops) - - // file-cache config - assert.Equal(t.T(), int64(100), mountConfig.FileCacheConfig.MaxSizeMB) - assert.True(t.T(), mountConfig.FileCacheConfig.CacheFileForRangeRead) - assert.True(t.T(), mountConfig.FileCacheConfig.EnableParallelDownloads) - assert.Equal(t.T(), 10, mountConfig.ParallelDownloadsPerFile) - assert.Equal(t.T(), -1, mountConfig.MaxParallelDownloads) - assert.Equal(t.T(), 100, mountConfig.DownloadChunkSizeMB) - assert.False(t.T(), mountConfig.FileCacheConfig.EnableCRC) - assert.Equal(t.T(), int64(8192), mountConfig.FileCacheConfig.WriteBufferSize) - assert.Equal(t.T(), false, mountConfig.FileCacheConfig.EnableODirect) - - // gcs-retries - assert.Equal(t.T(), int64(6), mountConfig.GCSRetries.MaxRetryAttempts) - - // metrics config - assert.Equal(t.T(), 8080, mountConfig.MetricsConfig.PrometheusPort) -} - -func (t *YamlParserTest) TestReadConfigFile_InvalidLogConfig() { - _, err := ParseConfigFile("testdata/invalid_log_config.yaml") - - assert.ErrorContains(t.T(), err, fmt.Sprintf(parseConfigFileErrMsgFormat, "log severity should be one of [trace, debug, info, warning, error, off]")) -} - -func (t *YamlParserTest) TestReadConfigFile_MetatadaCacheConfig_TtlNotSet() { - mountConfig, err := ParseConfigFile("testdata/metadata_cache_config_ttl-unset.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.EqualValues(t.T(), cfg.TtlInSecsUnsetSentinel, mountConfig.MetadataCacheConfig.TtlInSeconds) -} - -func (t *YamlParserTest) TestReadConfigFile_MetatadaCacheConfig_TypeCacheMaxSizeNotSet() { - mountConfig, err := ParseConfigFile("testdata/metadata_cache_config_type-cache-max-size-mb_unset.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.Equal(t.T(), DefaultTypeCacheMaxSizeMB, mountConfig.MetadataCacheConfig.TypeCacheMaxSizeMB) -} - -func (t *YamlParserTest) TestReadConfigFile_MetatadaCacheConfig_StatCacheSizeNotSet() { - mountConfig, err := ParseConfigFile("testdata/metadata_cache_config_stat-cache-max-size-mb_unset.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.EqualValues(t.T(), cfg.StatCacheMaxSizeMBUnsetSentinel, mountConfig.MetadataCacheConfig.StatCacheMaxSizeMB) -} - -func (t *YamlParserTest) TestReadConfigFile_GrpcClientConfig_invalidConnPoolSize() { - _, err := ParseConfigFile("testdata/gcs_connection/invalid_conn_pool_size.yaml") - - assert.ErrorContains(t.T(), err, "error parsing gcs-connection configs: the value of conn-pool-size can't be less than 1") -} - -func (t *YamlParserTest) TestReadConfigFile_GrpcClientConfig_unsetConnPoolSize() { - mountConfig, err := ParseConfigFile("testdata/gcs_connection/unset_conn_pool_size.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.Equal(t.T(), DefaultGrpcConnPoolSize, mountConfig.GCSConnection.GRPCConnPoolSize) -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_InvalidIgnoreInterruptsValue() { - _, err := ParseConfigFile("testdata/file_system_config/invalid_ignore_interrupts.yaml") - - assert.ErrorContains(t.T(), err, "error parsing config file: yaml: unmarshal errors:\n line 2: cannot unmarshal !!str `abc` into bool") -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_UnsetIgnoreInterruptsValue() { - mountConfig, err := ParseConfigFile("testdata/file_system_config/unset_ignore_interrupts.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.True(t.T(), mountConfig.FileSystemConfig.IgnoreInterrupts) -} - -func (t *YamlParserTest) TestReadConfigFile_GCSAuth_InvalidAnonymousAccessValue() { - _, err := ParseConfigFile("testdata/gcs_auth/invalid_anonymous_access.yaml") - - assert.ErrorContains(t.T(), err, "error parsing config file: yaml: unmarshal errors:\n line 2: cannot unmarshal !!str `abc` into bool") -} - -func (t *YamlParserTest) TestReadConfigFile_GCSAuth_UnsetAnonymousAccessValue() { - mountConfig, err := ParseConfigFile("testdata/gcs_auth/unset_anonymous_access.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.False(t.T(), mountConfig.GCSAuth.AnonymousAccess) -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_InvalidDisableParallelDirops() { - _, err := ParseConfigFile("testdata/file_system_config/invalid_disable_parallel_dirops.yaml") - - assert.ErrorContains(t.T(), err, "error parsing config file: yaml: unmarshal errors:\n line 2: cannot unmarshal !!int `-1` into bool") -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_UnsetDisableParallelDirops() { - mountConfig, err := ParseConfigFile("testdata/file_system_config/unset_disable_parallel_dirops.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.False(t.T(), mountConfig.FileSystemConfig.DisableParallelDirops) -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_UnsetKernelListCacheTtl() { - mountConfig, err := ParseConfigFile("testdata/file_system_config/unset_kernel_list_cache_ttl.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.Equal(t.T(), DefaultKernelListCacheTtlSeconds, mountConfig.FileSystemConfig.KernelListCacheTtlSeconds) -} - -func (t *YamlParserTest) TestReadConfigFile_FileSystemConfig_ValidKernelListCacheTtl() { - mountConfig, err := ParseConfigFile("testdata/file_system_config/valid_kernel_list_cache_ttl.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.Equal(t.T(), int64(10), mountConfig.FileSystemConfig.KernelListCacheTtlSeconds) -} - -func (t *YamlParserTest) TestReadConfigFile_MetricsConfig_UnsetPrometheusPort() { - mountConfig, err := ParseConfigFile("testdata/metrics_config/unset_prometheus_port.yaml") - - assert.NoError(t.T(), err) - assert.NotNil(t.T(), mountConfig) - assert.Equal(t.T(), 0, mountConfig.MetricsConfig.PrometheusPort) -} diff --git a/internal/fs/inode/dir_test.go b/internal/fs/inode/dir_test.go index 3af10c5875..5610d0efbe 100644 --- a/internal/fs/inode/dir_test.go +++ b/internal/fs/inode/dir_test.go @@ -22,7 +22,6 @@ import ( "testing" "time" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/internal/util" "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" @@ -95,7 +94,7 @@ func (p DirentSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } // This poses a challenge for writing unit tests for includeFoldersAsPrefixes. func (t *DirTest) resetInode(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool) { - t.resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, config.DefaultTypeCacheMaxSizeMB, typeCacheTTL) + t.resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, 4, typeCacheTTL) } func (t *DirTest) resetInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool, typeCacheMaxSizeMB int64, typeCacheTTL time.Duration) { @@ -146,7 +145,7 @@ func (t *DirTest) createDirInode(dirInodeName string) DirInode { &t.bucket, &t.clock, &t.clock, - config.DefaultTypeCacheMaxSizeMB, + 4, false, ) } @@ -680,7 +679,7 @@ func (t *DirTest) LookUpChild_TypeCacheEnabled() { typeCacheMaxSizeMB int64 typeCacheTTL time.Duration }{{ - typeCacheMaxSizeMB: config.DefaultTypeCacheMaxSizeMB, + typeCacheMaxSizeMB: 4, typeCacheTTL: time.Second, }, { typeCacheMaxSizeMB: -1, @@ -716,7 +715,7 @@ func (t *DirTest) LookUpChild_TypeCacheDisabled() { typeCacheMaxSizeMB: 0, typeCacheTTL: time.Second, }, { - typeCacheMaxSizeMB: config.DefaultTypeCacheMaxSizeMB, + typeCacheMaxSizeMB: 4, typeCacheTTL: 0, }} diff --git a/internal/fs/inode/hns_dir_test.go b/internal/fs/inode/hns_dir_test.go index 3bc21e4a03..c85d431399 100644 --- a/internal/fs/inode/hns_dir_test.go +++ b/internal/fs/inode/hns_dir_test.go @@ -23,7 +23,6 @@ import ( "time" "github.com/googlecloudplatform/gcsfuse/v2/internal/cache/metadata" - "github.com/googlecloudplatform/gcsfuse/v2/internal/config" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage" "github.com/googlecloudplatform/gcsfuse/v2/internal/storage/gcs" "github.com/jacobsa/fuse/fuseops" @@ -61,7 +60,7 @@ func (t *HNSDirTest) SetupTest() { } func (t *HNSDirTest) resetDirInode(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool) { - t.resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, config.DefaultTypeCacheMaxSizeMB, typeCacheTTL) + t.resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing, 4, typeCacheTTL) } func (t *HNSDirTest) resetDirInodeWithTypeCacheConfigs(implicitDirs, enableNonexistentTypeCache, enableManagedFoldersListing bool, typeCacheMaxSizeMB int64, typeCacheTTL time.Duration) { @@ -111,7 +110,7 @@ func (t *HNSDirTest) createDirInode(dirInodeName string) DirInode { &t.bucket, &t.fixedTime, &t.fixedTime, - config.DefaultTypeCacheMaxSizeMB, + 4, false, ) } diff --git a/main.go b/main.go index f260b65ea4..9ba069bcbb 100644 --- a/main.go +++ b/main.go @@ -21,8 +21,6 @@ package main import ( "log" - "os" - "strings" "github.com/googlecloudplatform/gcsfuse/v2/cmd" "github.com/googlecloudplatform/gcsfuse/v2/internal/logger" @@ -51,26 +49,5 @@ func main() { go perf.HandleCPUProfileSignals() go perf.HandleMemoryProfileSignals() - // TODO: Clean this up after we gain enough confidence on CLI-Config Parity changes. - disableViperConfigFlag := "disable-viper-config" - var newOsArgs []string - for _, arg := range os.Args { - if arg == "-"+disableViperConfigFlag || arg == "--"+disableViperConfigFlag || arg == "-"+disableViperConfigFlag+"=true" || arg == "--"+disableViperConfigFlag+"=true" { - err := os.Setenv(cmd.EnableViperConfigEnvVariable, "false") - if err != nil { - logger.Infof("error while setting "+cmd.EnableViperConfigEnvVariable+" environment variable: %v", err) - } - } - if !strings.Contains(arg, disableViperConfigFlag) { - newOsArgs = append(newOsArgs, arg) - } - } - os.Args = newOsArgs - - if strings.ToLower(os.Getenv(cmd.EnableViperConfigEnvVariable)) == "false" { - cmd.ExecuteLegacyMain() - return - } - - cmd.ExecuteNewMain() + cmd.ExecuteMountCmd() } diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 63b5aca434..0000000000 --- a/main_test.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2024 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "os" - "testing" - - "github.com/googlecloudplatform/gcsfuse/v2/cmd" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestLegacyMainExecution(t *testing.T) { - originalArgs := os.Args - originalEnvVar := os.Getenv("ENABLE_GCSFUSE_VIPER_CONFIG") - originalExecuteLegacyMain := cmd.ExecuteLegacyMain - defer func() { - // Restore original os.Args after the test. - os.Args = originalArgs - // Reset the environment variable. - _ = os.Setenv("ENABLE_GCSFUSE_VIPER_CONFIG", originalEnvVar) - // Restore original execute function. - cmd.ExecuteLegacyMain = originalExecuteLegacyMain - }() - - tests := []struct { - name string - inputArgs []string - inputEnvVariable string - expectedEnvVar string - expectedRemainingArgs []string - }{ - { - name: "disable_viper_config_with_short_flag", - inputArgs: []string{"gcsfuse", "-disable-viper-config", "bucket-name", "mount-point"}, - expectedEnvVar: "false", - expectedRemainingArgs: []string{"gcsfuse", "bucket-name", "mount-point"}, - }, - { - name: "disable_viper_config_with_posix_flag", - inputArgs: []string{"gcsfuse", "--disable-viper-config", "bucket-name", "mount-point"}, - expectedEnvVar: "false", - expectedRemainingArgs: []string{"gcsfuse", "bucket-name", "mount-point"}, - }, - { - name: "disable_viper_config_with_short_flag_and_value", - inputArgs: []string{"gcsfuse", "-disable-viper-config=true", "bucket-name", "mount-point"}, - expectedEnvVar: "false", - expectedRemainingArgs: []string{"gcsfuse", "bucket-name", "mount-point"}, - }, - { - name: "disable_viper_config_with_posix_flag_and_value", - inputArgs: []string{"gcsfuse", "--disable-viper-config=true", "bucket-name", "mount-point"}, - expectedEnvVar: "false", - expectedRemainingArgs: []string{"gcsfuse", "bucket-name", "mount-point"}, - }, - { - name: "disable_via_env_variable", - inputArgs: []string{"gcsfuse", "--implicit-dirs", "--debug_fuse", "bucket-name", "mount-point"}, - inputEnvVariable: "false", - expectedEnvVar: "false", - expectedRemainingArgs: []string{"gcsfuse", "--implicit-dirs", "--debug_fuse", "bucket-name", "mount-point"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - os.Args = tt.inputArgs - err := os.Setenv("ENABLE_GCSFUSE_VIPER_CONFIG", tt.inputEnvVariable) - require.NoError(t, err) - legacyMainCalled := false - cmd.ExecuteLegacyMain = func() { - legacyMainCalled = true - } - - main() - - assert.EqualValues(t, tt.expectedEnvVar, os.Getenv("ENABLE_GCSFUSE_VIPER_CONFIG")) - assert.EqualValues(t, tt.expectedRemainingArgs, os.Args) - assert.Equal(t, true, legacyMainCalled) - }) - } -} - -func TestNewMainExecution(t *testing.T) { - originalArgs := os.Args - originalEnvVar := os.Getenv("ENABLE_GCSFUSE_VIPER_CONFIG") - originalExecuteNewMain := cmd.ExecuteNewMain - defer func() { - // Restore original os.Args after the test. - os.Args = originalArgs - // Reset the environment variable. - _ = os.Setenv("ENABLE_GCSFUSE_VIPER_CONFIG", originalEnvVar) - // Restore original execute function. - cmd.ExecuteNewMain = originalExecuteNewMain - }() - - tests := []struct { - name string - inputArgs []string - inputEnvVariable string - expectedEnvVar string - expectedRemainingArgs []string - }{ - { - name: "no_disable_flag", - inputArgs: []string{"gcsfuse", "--implicit-dirs", "--debug_fuse", "bucket-name", "mount-point"}, - expectedEnvVar: "", // No change expected - expectedRemainingArgs: []string{"gcsfuse", "--implicit-dirs", "--debug_fuse", "bucket-name", "mount-point"}, - }, - { - name: "enable_via_env_variable", - inputArgs: []string{"gcsfuse", "--flag1", "bucket-name", "mount-point"}, - inputEnvVariable: "true", - expectedEnvVar: "true", - expectedRemainingArgs: []string{"gcsfuse", "--flag1", "bucket-name", "mount-point"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - os.Args = tt.inputArgs - err := os.Setenv("ENABLE_GCSFUSE_VIPER_CONFIG", tt.inputEnvVariable) - require.NoError(t, err) - newMainCalled := false - cmd.ExecuteNewMain = func() { - newMainCalled = true - } - - main() - - assert.EqualValues(t, tt.expectedEnvVar, os.Getenv("ENABLE_GCSFUSE_VIPER_CONFIG")) - assert.EqualValues(t, tt.expectedRemainingArgs, os.Args) - assert.Equal(t, true, newMainCalled) - }) - } -} diff --git a/tools/build_gcsfuse/main_test.go b/tools/build_gcsfuse/main_test.go index 2bfdec5eb6..e23cd06707 100644 --- a/tools/build_gcsfuse/main_test.go +++ b/tools/build_gcsfuse/main_test.go @@ -34,44 +34,31 @@ func TestVersion(t *testing.T) { t.Fatalf("Error while building binary: %v", err) } testCases := []struct { - name string - args string - expected string - enableViperConfig string + name string + args string + expected string }{ { - name: "Version Flag without Viper config", - args: "--version", - expected: "gcsfuse version 99.88.77", - enableViperConfig: "false", + name: "Version Flag", + args: "--version", + expected: "gcsfuse version 99.88.77", }, { - name: "Version Flag with Viper config", - args: "--version", - expected: "gcsfuse version 99.88.77", - enableViperConfig: "true", - }, - { - name: "Version Shorthand with Viper config", - args: "-v", - expected: "gcsfuse version 99.88.77", - enableViperConfig: "true", + name: "Version Shorthand with Viper config", + args: "-v", + expected: "gcsfuse version 99.88.77", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err = os.Setenv("ENABLE_GCSFUSE_VIPER_CONFIG", tc.enableViperConfig) - if err != nil { - t.Fatalf("Error while setting ENABLE_GCSFUSE_VIPER_CONFIG environment variable: %v", err) - } - cmd := exec.Command(path.Join(dir, "bin/gcsfuse"), tc.args) output, err := cmd.CombinedOutput() if err != nil { t.Fatalf("Error running gcsfuse with args %v: %v", tc.args, err) } + assert.Contains(t, string(output), tc.expected) }) } diff --git a/tools/integration_tests/mounting/gcsfuse_test.go b/tools/integration_tests/mounting/gcsfuse_test.go index 44360ba857..6eb4f7d94c 100644 --- a/tools/integration_tests/mounting/gcsfuse_test.go +++ b/tools/integration_tests/mounting/gcsfuse_test.go @@ -726,82 +726,3 @@ func (t *GcsfuseTest) BothLogAndKeyFilePath() { ExpectEq(nil, err) } } - -func (t *GcsfuseTest) TestDisableViperConfig() { - testCases := []struct { - name string - args []string - env string - }{ - { - name: "disable_viper_config_with_short_flag", - args: []string{"-disable-viper-config", "-metadata-cache-ttl-secs", "100"}, - env: "", - }, - { - name: "disable_viper_config_with_posix_flag", - args: []string{"--disable-viper-config", "--metadata-cache-ttl-secs", "100"}, - env: "", - }, - { - name: "disable_viper_config_with_short_flag_and_value", - args: []string{"-disable-viper-config=true", "-metadata-cache-ttl-secs=100"}, - env: "", - }, - { - name: "disable_viper_config_with_posix_flag_and_value", - args: []string{"--disable-viper-config=true", "--metadata-cache-ttl-secs=100"}, - env: "", - }, - { - name: "disable_via_env_variable", - args: []string{"--metadata-cache-ttl-secs=100"}, - env: "false", - }, - { - name: "env_true_flag_disabled", - args: []string{"--metadata-cache-ttl-secs=100", "--disable-viper-config=true"}, - env: "true", - }, - } - - for _, tc := range testCases { - args := append(tc.args, canned.FakeBucketName, t.dir) - env := []string{fmt.Sprintf("ENABLE_GCSFUSE_VIPER_CONFIG=%s", tc.env)} - - err := t.runGcsfuseWithEnv(args, env) - - // Expect that the mount fails as we are using a flag in viper config (which is disabled). - ExpectNe(nil, err) - } -} - -func (t *GcsfuseTest) TestEnabledViperConfig() { - testCases := []struct { - name string - args []string - env string - }{ - { - name: "no_disable_flag_or_env", - args: []string{"--metadata-cache-ttl-secs=100"}, - env: "", - }, - { - name: "enabled_via_env_variable", - args: []string{"--metadata-cache-ttl-secs=100"}, - env: "true", - }, - } - - for _, tc := range testCases { - args := append(tc.args, canned.FakeBucketName, t.dir) - env := []string{fmt.Sprintf("ENABLE_GCSFUSE_VIPER_CONFIG=%s", tc.env)} - - err := t.runGcsfuseWithEnv(args, env) - - ExpectEq(nil, err) - err = util.Unmount(t.dir) - ExpectEq(nil, err) - } -}