debug setup; not enabled for any container

development/mimir-microservices-mode/compose-up.sh

@@ -25,10 +25,10 @@ cd "$SCRIPT_DIR" && make
 CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/mimir "${SCRIPT_DIR}"/../../cmd/mimir
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" distributor-1
 
-if [ "$(yq '.services.query-tee' "${SCRIPT_DIR}"/docker-compose.yml)" != "null" ]; then
-  # If query-tee is enabled, build its binary and image as well.
-  CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/../../cmd/query-tee "${SCRIPT_DIR}"/../../cmd/query-tee
-  docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" query-tee
-fi
+#if [ "$(yq '.services.query-tee' "${SCRIPT_DIR}"/docker-compose.yml)" != "null" ]; then
+#  # If query-tee is enabled, build its binary and image as well.
+#  CGO_ENABLED=0 GOOS=linux go build -mod=vendor -tags=netgo,stringlabels -gcflags "all=-N -l" -o "${SCRIPT_DIR}"/../../cmd/query-tee "${SCRIPT_DIR}"/../../cmd/query-tee
+#  docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml build --build-arg BUILD_IMAGE="${BUILD_IMAGE}" query-tee
+#fi
 
 docker_compose -f "${SCRIPT_DIR}"/docker-compose.yml up "$@"

development/mimir-microservices-mode/config/mimir.yaml

@@ -138,6 +138,7 @@ frontend:
   cache_results: true
   additional_query_queue_dimensions_enabled: true
   shard_active_series_queries: true
+  max_outstanding_per_tenant: 10000
 
   # Uncomment when using "dns" service discovery mode for query-scheduler.
   # scheduler_address: "query-scheduler:9011"
@@ -163,6 +164,7 @@ query_scheduler:
   # Change to "dns" to switch to query-scheduler DNS-based service discovery.
   service_discovery_mode: "ring"
   additional_query_queue_dimensions_enabled: true
+  max_outstanding_requests_per_tenant: 10000
 
 limits:
   # Limit max query time range to 31d

development/mimir-microservices-mode/dev.dockerfile

@@ -1,11 +1,12 @@
 ARG BUILD_IMAGE # Use ./compose-up.sh to build this image.
 FROM $BUILD_IMAGE
 ENV CGO_ENABLED=0
-RUN go install github.com/go-delve/delve/cmd/dlv@v1.21.0
+RUN go install github.com/go-delve/delve/cmd/dlv@v1.22.1
 
 FROM alpine:3.19.1
 
 RUN     mkdir /mimir
 WORKDIR /mimir
 COPY     ./mimir ./
-COPY --from=0 /go/bin/dlv ./
+RUN ln -s ./mimir /usr/local/bin/mimir
+COPY --from=0 /go/bin/dlv  /usr/local/bin/dlv

development/mimir-microservices-mode/docker-compose.jsonnet

@@ -5,7 +5,7 @@ std.manifestYamlDoc({
 
     // If true, Mimir services are run under Delve debugger, that can be attached to via remote-debugging session.
     // Note that Delve doesn't forward signals to the Mimir process, so Mimir components don't shutdown cleanly.
-    debug: false,
+    debug: true,
 
     // How long should Mimir docker containers sleep before Mimir is started.
     sleep_seconds: 3,
@@ -23,7 +23,7 @@ std.manifestYamlDoc({
     ring: 'memberlist',
 
     // If true, a load generator is started.
-    enable_load_generator: true,
+    enable_load_generator: false,
 
     // If true, start and enable scraping by these components.
     // Note that if more than one component is enabled, the dashboards shown in Grafana may contain duplicate series or aggregates may be doubled or tripled.

integration/e2emimir/client.go

@@ -1523,7 +1523,7 @@ func (c *Client) doRequest(method, url string, body io.Reader) (*http.Response,
 	req.Header.Set("X-Scope-OrgID", c.orgID)
 
 	client := *c.httpClient
-	client.Timeout = c.timeout
+	client.Timeout = 2 * time.Minute
 
 	return client.Do(req)
 }

integration/e2emimir/service.go

@@ -22,12 +22,13 @@ func NewMimirService(
 	envVars map[string]string,
 	httpPort int,
 	grpcPort int,
-	otherPorts ...int,
+	otherPorts []int,
+	portMapContainerToLocal map[int]int,
 ) *MimirService {
 	s := &MimirService{
 		// We don't expose the gRPC port cause we don't need to access it from the host
 		// (exposing ports have a negative performance impact on starting/stopping containers).
-		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, otherPorts...),
+		HTTPService: e2e.NewHTTPService(name, image, command, readiness, httpPort, otherPorts, portMapContainerToLocal),
 		grpcPort:    grpcPort,
 	}
 	s.SetEnvVars(envVars)

integration/e2emimir/services.go

@@ -15,12 +15,14 @@ import (
 )
 
 const (
-	httpPort = 8080
-	grpcPort = 9095
+	httpPort        = 8080
+	grpcPort        = 9095
+	mimirBinaryName = "mimir"
+	delveBinaryName = "dlv"
 )
 
-// GetDefaultImage returns the Docker image to use to run Mimir.
-func GetDefaultImage() string {
+// GetMimirImageOrDefault returns the Docker image to use to run Mimir.
+func GetMimirImageOrDefault() string {
 	// Get the mimir image from the MIMIR_IMAGE env variable,
 	// falling back to grafana/mimir:latest"
 	if img := os.Getenv("MIMIR_IMAGE"); img != "" {
@@ -68,20 +70,56 @@ func getExtraFlags() map[string]string {
 	return extraArgs
 }
 
-func newMimirServiceFromOptions(name string, defaultFlags, flags map[string]string, options ...Option) *MimirService {
+func newMimirServiceFromOptions(name string, defaultFlags, overrideFlags map[string]string, options ...Option) *MimirService {
+	o := newOptions(options)
+	serviceFlags := o.MapFlags(e2e.MergeFlags(defaultFlags, overrideFlags, getExtraFlags()))
+
+	return NewMimirService(
+		name,
+		"grafana/mimir:latest",
+		e2e.NewCommandWithoutEntrypoint(mimirBinaryName, e2e.BuildArgs(serviceFlags)...),
+		e2e.NewHTTPReadinessProbe(o.HTTPPort, "/ready", 200, 299),
+		o.Environment,
+		o.HTTPPort,
+		o.GRPCPort,
+		o.OtherPorts,
+		nil,
+	)
+}
+
+func newMimirDebugServiceFromOptions(name string, defaultFlags, flags map[string]string, options ...Option) *MimirService {
 	o := newOptions(options)
 	serviceFlags := o.MapFlags(e2e.MergeFlags(defaultFlags, flags, getExtraFlags()))
-	binaryName := getBinaryNameForBackwardsCompatibility()
+	serviceArgs := e2e.BuildArgs(serviceFlags)
+
+	delveListenPort := o.HTTPPort + 10000 // follow convention in docker compose development environment
+
+	// delve args must be ordered correctly due to the use of `--`
+	// to delineate the end of delve args and beginning of the binary's args;
+	// do not use any interface utilizing a map structure as order will not be preserved
+	delveArgs := []string{
+		"--log",
+		"exec",
+		"mimir",
+		fmt.Sprintf("--listen=:%d", delveListenPort),
+		"--headless=true",
+		"--api-version=2",
+		"--accept-multiclient",
+		"--",
+	}
+
+	cmd := e2e.NewCommandWithoutEntrypoint(delveBinaryName, append(delveArgs, serviceArgs...)...)
 
 	return NewMimirService(
 		name,
 		o.Image,
-		e2e.NewCommandWithoutEntrypoint(binaryName, e2e.BuildArgs(serviceFlags)...),
+		cmd,
 		e2e.NewHTTPReadinessProbe(o.HTTPPort, "/ready", 200, 299),
 		o.Environment,
 		o.HTTPPort,
 		o.GRPCPort,
-		o.OtherPorts...,
+		[]int{delveListenPort},
+		map[int]int{delveListenPort: delveListenPort, o.HTTPPort: o.HTTPPort},
 	)
 }
 
@@ -95,10 +133,8 @@ func NewDistributor(name string, consulAddress string, flags map[string]string,
 			"-ingester.ring.replication-factor": "1",
 			"-distributor.remote-timeout":       "2s", // Fail fast in integration tests.
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Configure the distributor ring backend
 			"-distributor.ring.store": "memberlist",
 		},
@@ -115,10 +151,8 @@ func NewQuerier(name string, consulAddress string, flags map[string]string, opti
 			"-log.level":                        "warn",
 			"-ingester.ring.replication-factor": "1",
 			// Ingesters ring backend.
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Query-frontend worker.
 			"-querier.frontend-client.backoff-min-period": "100ms",
 			"-querier.frontend-client.backoff-max-period": "100ms",
@@ -160,10 +194,8 @@ func NewIngester(name string, consulAddress string, flags map[string]string, opt
 			"-log.level":                "warn",
 			"-ingester.ring.num-tokens": "512",
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 			// Speed up the startup.
 			"-ingester.ring.min-ready-duration": "0s",
 			// Enable native histograms
@@ -174,10 +206,6 @@ func NewIngester(name string, consulAddress string, flags map[string]string, opt
 	)
 }
 
-func getBinaryNameForBackwardsCompatibility() string {
-	return "mimir"
-}
-
 func NewQueryFrontend(name string, flags map[string]string, options ...Option) *MimirService {
 	return newMimirServiceFromOptions(
 		name,
@@ -186,8 +214,6 @@ func NewQueryFrontend(name string, flags map[string]string, options ...Option) *
 			"-log.level": "warn",
 			// Quickly detect query-scheduler when running it.
 			"-query-frontend.scheduler-dns-lookup-period": "1s",
-			// Always exercise remote read limits in integration tests.
-			"-query-frontend.remote-read-limits-enabled": "true",
 		},
 		flags,
 		options...,
@@ -225,20 +251,31 @@ func NewCompactor(name string, consulAddress string, flags map[string]string, op
 	)
 }
 
-func NewSingleBinary(name string, flags map[string]string, options ...Option) *MimirService {
+var singleBinaryDefaultFlags = map[string]string{
+	// Do not pass any extra default flags (except few used to speed up the test)
+	// because the config could be driven by the config file.
+	"-target":    "all",
+	"-log.level": "warn",
+	// Speed up the startup.
+	"-ingester.ring.min-ready-duration": "0s",
+	// Enable native histograms
+	"-ingester.native-histograms-ingestion-enabled": "true",
+}
+
+func NewSingleBinary(name string, overrideFlags map[string]string, options ...Option) *MimirService {
 	return newMimirServiceFromOptions(
 		name,
-		map[string]string{
-			// Do not pass any extra default flags (except few used to speed up the test)
-			// because the config could be driven by the config file.
-			"-target":    "all",
-			"-log.level": "warn",
-			// Speed up the startup.
-			"-ingester.ring.min-ready-duration": "0s",
-			// Enable native histograms
-			"-ingester.native-histograms-ingestion-enabled": "true",
-		},
-		flags,
+		singleBinaryDefaultFlags,
+		overrideFlags,
+		options...,
+	)
+}
+
+func NewDebugSingleBinary(name string, overrideFlags map[string]string, options ...Option) *MimirService {
+	return newMimirDebugServiceFromOptions(
+		name,
+		singleBinaryDefaultFlags,
+		overrideFlags,
 		options...,
 	)
 }
@@ -304,17 +341,17 @@ func NewAlertmanagerWithTLS(name string, flags map[string]string, options ...Opt
 		"-target":    "alertmanager",
 		"-log.level": "warn",
 	}, flags, getExtraFlags()))
-	binaryName := getBinaryNameForBackwardsCompatibility()
 
 	return NewMimirService(
 		name,
-		o.Image,
-		e2e.NewCommandWithoutEntrypoint(binaryName, e2e.BuildArgs(serviceFlags)...),
+		"grafana/mimir:latest",
+		e2e.NewCommandWithoutEntrypoint(mimirBinaryName, e2e.BuildArgs(serviceFlags)...),
 		e2e.NewTCPReadinessProbe(o.HTTPPort),
 		o.Environment,
 		o.HTTPPort,
 		o.GRPCPort,
-		o.OtherPorts...,
+		o.OtherPorts,
+		nil,
 	)
 }
 
@@ -325,10 +362,8 @@ func NewRuler(name string, consulAddress string, flags map[string]string, option
 			"-target":    "ruler",
 			"-log.level": "warn",
 			// Configure the ingesters ring backend
-			"-ingester.ring.store":                     "consul",
-			"-ingester.ring.consul.hostname":           consulAddress,
-			"-ingester.partition-ring.store":           "consul",
-			"-ingester.partition-ring.consul.hostname": consulAddress,
+			"-ingester.ring.store":           "consul",
+			"-ingester.ring.consul.hostname": consulAddress,
 		},
 		flags,
 		options...,
@@ -363,7 +398,7 @@ type Option func(*Options)
 // newOptions creates an Options with default values and applies the options provided.
 func newOptions(options []Option) *Options {
 	o := &Options{
-		Image:       GetDefaultImage(),
+		Image:       GetMimirImageOrDefault(),
 		MapFlags:    NoopFlagMapper,
 		HTTPPort:    httpPort,
 		GRPCPort:    grpcPort,
@@ -417,7 +452,7 @@ func WithConfigFile(configFile string) Option {
 }
 
 // WithNoopOption returns an option that doesn't change anything.
-func WithNoopOption() Option { return func(*Options) {} }
+func WithNoopOption() Option { return func(options *Options) {} }
 
 // FlagMapper is the type of function that maps flags, just to reduce some verbosity.
 type FlagMapper func(flags map[string]string) map[string]string

integration/integration_memberlist_single_binary_test.go

@@ -138,7 +138,7 @@ func newSingleBinary(name string, servername string, join string, testFlags map[
 		flags["-ingester.ring.observe-period"] = "0s" // No need to observe tokens because we're going to be the first instance.
 	}
 
-	serv := e2emimir.NewSingleBinary(
+	serv := e2emimir.NewDebugSingleBinary(
 		name,
 		mergeFlags(
 			DefaultSingleBinaryFlags(),

integration/vault_test.go

@@ -30,6 +30,8 @@ func TestVaultTokenRenewal(t *testing.T) {
 		nil,
 		e2e.NewHTTPReadinessProbe(httpPort, "/v1/sys/health", 200, 200),
 		httpPort,
+		nil,
+		nil,
 	)
 	vault.SetEnvVars(map[string]string{"VAULT_DEV_ROOT_TOKEN_ID": devToken})
 	require.NoError(t, s.StartAndWaitReady(vault))

pkg/scheduler/queue/tenant_querier_assignment.go

@@ -213,6 +213,24 @@ func (tqa *tenantQuerierAssignments) addQuerierConnection(querierID QuerierID) (
 	return tqa.recomputeTenantQueriers()
 }
 
+func (tqa *tenantQuerierAssignments) removeTenantOld(tenantID TenantID) {
+	tenant := tqa.tenantsByID[tenantID]
+	if tenant == nil {
+		return
+	}
+	delete(tqa.tenantsByID, tenantID)
+	tqa.tenantIDOrder[tenant.orderIndex] = emptyTenantID
+
+	// Shrink tenant list if possible by removing empty tenant IDs.
+	// We remove only from the end; removing from the middle would re-index all tenant IDs
+	// and skip tenants when starting iteration from a querier-provided lastTenantIndex.
+	// Empty tenant IDs stuck in the middle of the slice are handled
+	// by replacing them when a new tenant ID arrives in the queue.
+	for i := len(tqa.tenantIDOrder) - 1; i >= 0 && tqa.tenantIDOrder[i] == emptyTenantID; i-- {
+		tqa.tenantIDOrder = tqa.tenantIDOrder[:i]
+	}
+}
+
 // removeTenant only manages deletion of a *queueTenant from tenantsByID. All other
 // tenant deletion (e.g., from tenantIDOrder, or tenantNodes) is done during the dequeue operation,
 // as we cannot remove from those things arbitrarily; we must check whether other tenant

pkg/scheduler/queue/tenant_queues.go

@@ -206,7 +206,7 @@ func (qb *queueBroker) dequeueRequestForQuerier(
 		queueNodeAfterDequeue := tq.getNode(queuePath)
 		if queueNodeAfterDequeue == nil {
 			// queue node was deleted due to being empty after dequeue
-			qb.tenantQuerierAssignments.removeTenant(tenant.tenantID)
+			qb.tenantQuerierAssignments.removeTenantOld(tenant.tenantID)
 		}
 	} else if itq, ok := qb.tree.(*MultiQueuingAlgorithmTreeQueue); ok {
 		queueNodeAfterDequeue := itq.GetNode(queuePath)