francoposa/casie tree queue sketch integration debug

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## main / unreleased
+* [ENHANCEMENT] Query-scheduler: Introduce `query-scheduler.prioritize-query-components`, which allows configuration of the request queue tree to prioritize dequeuing from a specific query component more highly than dequeueing from a specific tenant. #7873
 
 ### Grafana Mimir
 

cmd/mimir/config-descriptor.json

@@ -15909,6 +15909,17 @@
           "fieldType": "boolean",
           "fieldCategory": "experimental"
         },
+        {
+          "kind": "field",
+          "name": "use_multi_algorithm_query_queue",
+          "required": false,
+          "desc": "Use an experimental version of the query queue which has the same behavior as the existing queue, but integrates tenant selection into the tree model.",
+          "fieldValue": null,
+          "fieldDefaultValue": false,
+          "fieldFlag": "query-scheduler.use-multi-algorithm-query-queue",
+          "fieldType": "boolean",
+          "fieldCategory": "experimental"
+        },
         {
           "kind": "field",
           "name": "querier_forget_delay",

cmd/mimir/help-all.txt.tmpl

@@ -2317,6 +2317,8 @@ Usage of ./cmd/mimir/mimir:
     	Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi. (default "memberlist")
   -query-scheduler.service-discovery-mode string
     	[experimental] Service discovery mode that query-frontends and queriers use to find query-scheduler instances. When query-scheduler ring-based service discovery is enabled, this option needs be set on query-schedulers, query-frontends and queriers. Supported values are: dns, ring. (default "dns")
+  -query-scheduler.use-multi-algorithm-query-queue
+    	[experimental] Use an experimental version of the query queue which has the same behavior as the existing queue, but integrates tenant selection into the tree model.
   -ruler-storage.azure.account-key string
     	Azure storage account key. If unset, Azure managed identities will be used for authentication instead.
   -ruler-storage.azure.account-name string

development/mimir-microservices-mode/compose-up.sh

@@ -25,10 +25,10 @@ cd "$SCRIPT_DIR" && make
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/mimir "${SCRIPT_DIR}"/../../cmd/mimir
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" distributor-1
 
-if [ "$(yq '.services.query-tee' "${SCRIPT_DIR}"/docker-compose.yml)" != "null" ]; then
-  # If query-tee is enabled, build its binary and image as well.
-  CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/../../cmd/query-tee "${SCRIPT_DIR}"/../../cmd/query-tee
-  docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" query-tee
-fi
+#if [ "$(yq '.services.query-tee' "${SCRIPT_DIR}"/docker-compose.yml)" != "null" ]; then
+#  # If query-tee is enabled, build its binary and image as well.
+#  CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/../../cmd/query-tee "${SCRIPT_DIR}"/../../cmd/query-tee
+#  docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" query-tee
+#fi
 
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"

development/mimir-microservices-mode/config/mimir.yaml

@@ -138,6 +138,7 @@ frontend:
   cache_results: true
   additional_query_queue_dimensions_enabled: true
   shard_active_series_queries: true
+  max_outstanding_per_tenant: 10000
 
   # Uncomment when using "dns" service discovery mode for query-scheduler.
   # scheduler_address: "query-scheduler:9011"
@@ -163,6 +164,7 @@ query_scheduler:
   # Change to "dns" to switch to query-scheduler DNS-based service discovery.
   service_discovery_mode: "ring"
   additional_query_queue_dimensions_enabled: true
+  max_outstanding_requests_per_tenant: 10000
 
 limits:
   # Limit max query time range to 31d

development/mimir-microservices-mode/dev.dockerfile

@@ -1,11 +1,12 @@
 ARG BUILD_IMAGE # Use ./compose-up.sh to build this image.
 FROM $BUILD_IMAGE
 ENV CGO_ENABLED=0
-RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.0
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
 
 FROM alpine:3.19.1
 
 RUN     mkdir /mimir
 WORKDIR /mimir
 COPY     ./mimir ./
-COPY --from=0 /go/bin/dlv ./
+RUN ln -s ./mimir /usr/local/bin/mimir
+COPY --from=0 /go/bin/dlv  /usr/local/bin/dlv

development/mimir-microservices-mode/docker-compose.jsonnet

@@ -5,7 +5,7 @@ std.manifestYamlDoc({
 
     // If true, Mimir services are run under Delve debugger, that can be attached to via remote-debugging session.
     // Note that Delve doesn't forward signals to the Mimir process, so Mimir components don't shutdown cleanly.
-    debug: false,
+    debug: true,
 
     // How long should Mimir docker containers sleep before Mimir is started.
     sleep_seconds: 3,
@@ -23,7 +23,7 @@ std.manifestYamlDoc({
     ring: 'memberlist',
 
     // If true, a load generator is started.
-    enable_load_generator: true,
+    enable_load_generator: false,
 
     // If true, start and enable scraping by these components.
     // Note that if more than one component is enabled, the dashboards shown in Grafana may contain duplicate series or aggregates may be doubled or tripled.

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -1708,6 +1708,12 @@ The `query_scheduler` block configures the query-scheduler.
 # CLI flag: -query-scheduler.additional-query-queue-dimensions-enabled
 [additional_query_queue_dimensions_enabled: <boolean> | default = false]
 
+# (experimental) Use an experimental version of the query queue which has the
+# same behavior as the existing queue, but integrates tenant selection into the
+# tree model.
+# CLI flag: -query-scheduler.use-multi-algorithm-query-queue
+[use_multi_algorithm_query_queue: <boolean> | default = false]
+
 # (experimental) If a querier disconnects without sending notification about
 # graceful shutdown, the query-scheduler will keep the querier in the tenant's
 # shard until the forget delay has passed. This feature is useful to reduce the

integration/e2emimir/client.go

@@ -1523,7 +1523,7 @@ func (c *Client) doRequest(method, url string, body io.Reader) (*http.Response,
 	req.Header.Set("X-Scope-OrgID", c.orgID)
 
 	client := *c.httpClient
-	client.Timeout = c.timeout
+	client.Timeout = 2 * time.Minute
 
 	return client.Do(req)
 }

integration/e2emimir/service.go

@@ -22,12 +22,13 @@ func NewMimirService(
 	envVars map[string]string,
 	httpPort int,
 	grpcPort int,
-	otherPorts ...int,
+	otherPorts []int,
+	portMapContainerToLocal map[int]int,
 ) *MimirService {
 	s := &MimirService{
 		// We don't expose the gRPC port cause we don't need to access it from the host
 		// (exposing ports have a negative performance impact on starting/stopping containers).
-		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, otherPorts...),
+		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, otherPorts, portMapContainerToLocal),
 		grpcPort:    grpcPort,
 	}
 	s.SetEnvVars(envVars)

integration/e2emimir/services.go

@@ -15,12 +15,14 @@ import (
 )
 
 const (
-	httpPort = 8080
-	grpcPort = 9095
+	httpPort        = 8080
+	grpcPort        = 9095
+	mimirBinaryName = "mimir"
+	delveBinaryName = "dlv"
 )
 
-// GetDefaultImage returns the Docker image to use to run Mimir.
-func GetDefaultImage() string {
+// GetMimirImageOrDefault returns the Docker image to use to run Mimir.
+func GetMimirImageOrDefault() string {
 	// Get the mimir image from the MIMIR_IMAGE env variable,
 	// falling back to grafana/mimir:latest"
 	if img := os.Getenv("MIMIR_IMAGE"); img != "" {
@@ -68,20 +70,56 @@ func getExtraFlags() map[string]string {
 	return extraArgs
 }
 
-func newMimirServiceFromOptions(name string, defaultFlags, flags map[string]string, options ...Option) *MimirService {
+func newMimirServiceFromOptions(name string, defaultFlags, overrideFlags map[string]string, options ...Option) *MimirService {
+	o := newOptions(options)
+	serviceFlags := o.MapFlags(e2e.MergeFlags(defaultFlags, overrideFlags, getExtraFlags()))
+
+	return NewMimirService(
+		name,
+		"grafana/mimir:latest",
+		e2e.NewCommandWithoutEntrypoint(mimirBinaryName, e2e.BuildArgs(serviceFlags)...),
+		e2e.NewHTTPReadinessProbe(o.HTTPPort, "/ready", 200, 299),
+		o.Environment,
+		o.HTTPPort,
+		o.GRPCPort,
+		o.OtherPorts,
+		nil,
+	)
+}
+
+func newMimirDebugServiceFromOptions(name string, defaultFlags, flags map[string]string, options ...Option) *MimirService {
 	o := newOptions(options)
 	serviceFlags := o.MapFlags(e2e.MergeFlags(defaultFlags, flags, getExtraFlags()))
-	binaryName := getBinaryNameForBackwardsCompatibility()
+	serviceArgs := e2e.BuildArgs(serviceFlags)
+
+	delveListenPort := o.HTTPPort + 10000 // follow convention in docker compose development environment
+
+	// delve args must be ordered correctly due to the use of `--`
+	// to delineate the end of delve args and beginning of the binary's args;
+	// do not use any interface utilizing a map structure as order will not be preserved
+	delveArgs := []string{
+		"--log",
+		"exec",
+		"mimir",
+		fmt.Sprintf("--listen=:%d", delveListenPort),
+		"--headless=true",
+		"--api-version=2",
+		"--accept-multiclient",
+		"--",
+	}
+
+	cmd := e2e.NewCommandWithoutEntrypoint(delveBinaryName, append(delveArgs, serviceArgs...)...)
 
 	return NewMimirService(
 		name,
 		o.Image,
-		e2e.NewCommandWithoutEntrypoint(binaryName, e2e.BuildArgs(serviceFlags)...),
+		cmd,
 		e2e.NewHTTPReadinessProbe(o.HTTPPort, "/ready", 200, 299),
 		o.Environment,
 		o.HTTPPort,
 		o.GRPCPort,
-		o.OtherPorts...,
+		[]int{delveListenPort},
+		map[int]int{delveListenPort: delveListenPort, o.HTTPPort: o.HTTPPort},
 	)
 }
 
@@ -95,10 +133,8 @@ func NewDistributor(name string, consulAddress string, flags map[string]string,
 			"-ingester.ring.replication-factor": "1",
 			"-distributor.remote-timeout":       "2s", // Fail fast in integration tests.
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Configure the distributor ring backend
 			"-distributor.ring.store": "memberlist",
 		},
@@ -115,10 +151,8 @@ func NewQuerier(name string, consulAddress string, flags map[string]string, opti
 			"-log.level":                        "warn",
 			"-ingester.ring.replication-factor": "1",
 			// Ingesters ring backend.
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Query-frontend worker.
 			"-querier.frontend-client.backoff-min-period": "100ms",
 			"-querier.frontend-client.backoff-max-period": "100ms",
@@ -160,10 +194,8 @@ func NewIngester(name string, consulAddress string, flags map[string]string, opt
 			"-log.level":                "warn",
 			"-ingester.ring.num-tokens": "512",
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Speed up the startup.
 			"-ingester.ring.min-ready-duration": "0s",
 			// Enable native histograms
@@ -174,10 +206,6 @@ func NewIngester(name string, consulAddress string, flags map[string]string, opt
 	)
 }
 
-func getBinaryNameForBackwardsCompatibility() string {
-	return "mimir"
-}
-
 func NewQueryFrontend(name string, flags map[string]string, options ...Option) *MimirService {
 	return newMimirServiceFromOptions(
 		name,
@@ -186,8 +214,6 @@ func NewQueryFrontend(name string, flags map[string]string, options ...Option) *
 			"-log.level": "warn",
 			// Quickly detect query-scheduler when running it.
 			"-query-frontend.scheduler-dns-lookup-period": "1s",
-			// Always exercise remote read limits in integration tests.
-			"-query-frontend.remote-read-limits-enabled": "true",
 		},
 		flags,
 		options...,
@@ -225,20 +251,31 @@ func NewCompactor(name string, consulAddress string, flags map[string]string, op
 	)
 }
 
-func NewSingleBinary(name string, flags map[string]string, options ...Option) *MimirService {
+var singleBinaryDefaultFlags = map[string]string{
+	// Do not pass any extra default flags (except few used to speed up the test)
+	// because the config could be driven by the config file.
+	"-target":    "all",
+	"-log.level": "warn",
+	// Speed up the startup.
+	"-ingester.ring.min-ready-duration": "0s",
+	// Enable native histograms
+	"-ingester.native-histograms-ingestion-enabled": "true",
+}
+
+func NewSingleBinary(name string, overrideFlags map[string]string, options ...Option) *MimirService {
 	return newMimirServiceFromOptions(
 		name,
-		map[string]string{
-			// Do not pass any extra default flags (except few used to speed up the test)
-			// because the config could be driven by the config file.
-			"-target":    "all",
-			"-log.level": "warn",
-			// Speed up the startup.
-			"-ingester.ring.min-ready-duration": "0s",
-			// Enable native histograms
-			"-ingester.native-histograms-ingestion-enabled": "true",
-		},
-		flags,
+		singleBinaryDefaultFlags,
+		overrideFlags,
+		options...,
+	)
+}
+
+func NewDebugSingleBinary(name string, overrideFlags map[string]string, options ...Option) *MimirService {
+	return newMimirDebugServiceFromOptions(
+		name,
+		singleBinaryDefaultFlags,
+		overrideFlags,
 		options...,
 	)
 }
@@ -304,17 +341,17 @@ func NewAlertmanagerWithTLS(name string, flags map[string]string, options ...Opt
 		"-target":    "alertmanager",
 		"-log.level": "warn",
 	}, flags, getExtraFlags()))
-	binaryName := getBinaryNameForBackwardsCompatibility()
 
 	return NewMimirService(
 		name,
-		o.Image,
-		e2e.NewCommandWithoutEntrypoint(binaryName, e2e.BuildArgs(serviceFlags)...),
+		"grafana/mimir:latest",
+		e2e.NewCommandWithoutEntrypoint(mimirBinaryName, e2e.BuildArgs(serviceFlags)...),
 		e2e.NewTCPReadinessProbe(o.HTTPPort),
 		o.Environment,
 		o.HTTPPort,
 		o.GRPCPort,
-		o.OtherPorts...,
+		o.OtherPorts,
+		nil,
 	)
 }
 
@@ -325,10 +362,8 @@ func NewRuler(name string, consulAddress string, flags map[string]string, option
 			"-target":    "ruler",
 			"-log.level": "warn",
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 		},
 		flags,
 		options...,
@@ -363,7 +398,7 @@ type Option func(*Options)
 // newOptions creates an Options with default values and applies the options provided.
 func newOptions(options []Option) *Options {
 	o := &Options{
-		Image:       GetDefaultImage(),
+		Image:       GetMimirImageOrDefault(),
 		MapFlags:    NoopFlagMapper,
 		HTTPPort:    httpPort,
 		GRPCPort:    grpcPort,
@@ -417,7 +452,7 @@ func WithConfigFile(configFile string) Option {
 }
 
 // WithNoopOption returns an option that doesn't change anything.
-func WithNoopOption() Option { return func(*Options) {} }
+func WithNoopOption() Option { return func(options *Options) {} }
 
 // FlagMapper is the type of function that maps flags, just to reduce some verbosity.
 type FlagMapper func(flags map[string]string) map[string]string

integration/integration_memberlist_single_binary_test.go

@@ -138,7 +138,7 @@ func newSingleBinary(name string, servername string, join string, testFlags map[
 		flags["-ingester.ring.observe-period"] = "0s" // No need to observe tokens because we're going to be the first instance.
 	}
 
-	serv := e2emimir.NewSingleBinary(
+	serv := e2emimir.NewDebugSingleBinary(
 		name,
 		mergeFlags(
 			DefaultSingleBinaryFlags(),

integration/vault_test.go

@@ -30,6 +30,8 @@ func TestVaultTokenRenewal(t *testing.T) {
 		nil,
 		e2e.NewHTTPReadinessProbe(httpPort, "/v1/sys/health", 200, 200),
 		httpPort,
+		nil,
+		nil,
 	)
 	vault.SetEnvVars(map[string]string{"VAULT_DEV_ROOT_TOKEN_ID": devToken})
 	require.NoError(t, s.StartAndWaitReady(vault))