Floors: Dynamic fetch

config/account.go

@@ -58,17 +58,29 @@ type AccountCCPA struct {
 }
 
 type AccountPriceFloors struct {
-	Enabled                bool `mapstructure:"enabled" json:"enabled"`
-	EnforceFloorsRate      int  `mapstructure:"enforce_floors_rate" json:"enforce_floors_rate"`
-	AdjustForBidAdjustment bool `mapstructure:"adjust_for_bid_adjustment" json:"adjust_for_bid_adjustment"`
-	EnforceDealFloors      bool `mapstructure:"enforce_deal_floors" json:"enforce_deal_floors"`
-	UseDynamicData         bool `mapstructure:"use_dynamic_data" json:"use_dynamic_data"`
-	MaxRule                int  `mapstructure:"max_rules" json:"max_rules"`
-	MaxSchemaDims          int  `mapstructure:"max_schema_dims" json:"max_schema_dims"`
+	Enabled                bool              `mapstructure:"enabled" json:"enabled"`
+	EnforceFloorsRate      int               `mapstructure:"enforce_floors_rate" json:"enforce_floors_rate"`
+	AdjustForBidAdjustment bool              `mapstructure:"adjust_for_bid_adjustment" json:"adjust_for_bid_adjustment"`
+	EnforceDealFloors      bool              `mapstructure:"enforce_deal_floors" json:"enforce_deal_floors"`
+	UseDynamicData         bool              `mapstructure:"use_dynamic_data" json:"use_dynamic_data"`
+	MaxRule                int               `mapstructure:"max_rules" json:"max_rules"`
+	MaxSchemaDims          int               `mapstructure:"max_schema_dims" json:"max_schema_dims"`
+	Fetch                  AccountFloorFetch `mapstructure:"fetch" json:"fetch"`
 }
 
-func (pf *AccountPriceFloors) validate(errs []error) []error {
+// Dynamic price floors fetching configuration
+type AccountFloorFetch struct {
+	Enabled     bool   `mapstructure:"enabled" json:"enabled"`
+	URL         string `mapstructure:"url" json:"url"`
+	Timeout     int    `mapstructure:"timeout_ms" json:"timeout_ms"`
+	MaxFileSize int    `mapstructure:"max_file_size_kb" json:"max_file_size_kb"`
+	MaxRules    int    `mapstructure:"max_rules" json:"max_rules"`
+	MaxAge      int    `mapstructure:"max_age_sec" json:"max_age_sec"`
+	Period      int    `mapstructure:"period_sec" json:"period_sec"`
+	AccountID   string `mapstructure:"accountID" json:"accountID"`
+}
 
+func (pf *AccountPriceFloors) validate(errs []error) []error {
 	if pf.EnforceFloorsRate < 0 || pf.EnforceFloorsRate > 100 {
 		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.enforce_floors_rate should be between 0 and 100`))
 	}
@@ -81,6 +93,30 @@ func (pf *AccountPriceFloors) validate(errs []error) []error {
 		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.max_schema_dims should be between 0 and 20`))
 	}
 
+	if pf.Fetch.Period > pf.Fetch.MaxAge {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.period_sec should be less than account_defaults.price_floors.fetch.max_age_sec`))
+	}
+
+	if pf.Fetch.Period < 300 {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.period_sec should not be less than 300 seconds`))
+	}
+
+	if !(pf.Fetch.MaxAge >= 600 && pf.Fetch.MaxAge < math.MaxInt32) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_age_sec should not be less than 600 seconds and greater than maximum integer value`))
+	}
+
+	if !(pf.Fetch.Timeout > 10 && pf.Fetch.Timeout < 10000) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.timeout_ms should be between 10 to 10,000 mili seconds`))
+	}
+
+	if !(pf.Fetch.MaxRules >= 0 && pf.Fetch.MaxRules < math.MaxInt32) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_rules should not be less than 0 seconds and greater than maximum integer value`))
+	}
+
+	if !(pf.Fetch.MaxFileSize >= 0 && pf.Fetch.MaxFileSize < math.MaxInt32) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_file_size_kb should not be less than 0 seconds and greater than maximum integer value`))
+	}
+
 	return errs
 }
 

config/account_test.go

@@ -871,36 +871,147 @@ func TestAccountPriceFloorsValidate(t *testing.T) {
 				EnforceFloorsRate: 100,
 				MaxRule:           200,
 				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 		},
 		{
 			description: "Invalid configuration: EnforceFloorRate:110",
 			pf: &AccountPriceFloors{
 				EnforceFloorsRate: 110,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.enforce_floors_rate should be between 0 and 100")},
 		},
 		{
 			description: "Invalid configuration: EnforceFloorRate:-10",
 			pf: &AccountPriceFloors{
 				EnforceFloorsRate: -10,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.enforce_floors_rate should be between 0 and 100")},
 		},
 		{
 			description: "Invalid configuration: MaxRule:-20",
 			pf: &AccountPriceFloors{
 				MaxRule: -20,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.max_rules should be between 0 and 2147483647")},
 		},
 		{
 			description: "Invalid configuration: MaxSchemaDims:100",
 			pf: &AccountPriceFloors{
 				MaxSchemaDims: 100,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.max_schema_dims should be between 0 and 20")},
 		},
+		{
+			description: "Invalid period for fetch",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:  100,
+					MaxAge:  600,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.period_sec should not be less than 300 seconds")},
+		},
+		{
+			description: "Invalid max age for fetch",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  500,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_age_sec should not be less than 600 seconds and greater than maximum integer value")},
+		},
+		{
+			description: "Period is greater than max age",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:  700,
+					MaxAge:  600,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.period_sec should be less than account_defaults.price_floors.fetch.max_age_sec")},
+		},
+		{
+			description: "Invalid timeout",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 4,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.timeout_ms should be between 10 to 10,000 mili seconds")},
+		},
+		{
+			description: "Invalid Max Rules",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:   300,
+					MaxAge:   600,
+					Timeout:  12,
+					MaxRules: -2,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_rules should not be less than 0 seconds and greater than maximum integer value")},
+		},
+		{
+			description: "Invalid Max File size",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetch: AccountFloorFetch{
+					Period:      300,
+					MaxAge:      600,
+					Timeout:     12,
+					MaxFileSize: -1,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_file_size_kb should not be less than 0 seconds and greater than maximum integer value")},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {

config/config.go

@@ -109,7 +109,13 @@ type Configuration struct {
 }
 
 type PriceFloors struct {
-	Enabled bool `mapstructure:"enabled"`
+	Enabled bool              `mapstructure:"enabled"`
+	Fetcher PriceFloorFetcher `mapstructure:"fetcher"`
+}
+
+type PriceFloorFetcher struct {
+	Worker   int `mapstructure:"worker"`
+	Capacity int `mapstructure:"capacity"`
 }
 
 const MIN_COOKIE_SIZE_BYTES = 500
@@ -1020,8 +1026,16 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
 	v.SetDefault("account_defaults.price_floors.use_dynamic_data", false)
 	v.SetDefault("account_defaults.price_floors.max_rules", 100)
 	v.SetDefault("account_defaults.price_floors.max_schema_dims", 3)
-	v.SetDefault("account_defaults.events_enabled", false)
+	v.SetDefault("account_defaults.price_floors.fetch.enabled", false)
+	v.SetDefault("account_defaults.price_floors.fetch.timeout_ms", 100)
+	v.SetDefault("account_defaults.price_floors.fetch.period_sec", 300)
+	v.SetDefault("account_defaults.price_floors.fetch.max_age_sec", 600)
+
+	//Defaults for Price floor fetcher
+	v.SetDefault("price_floor.fetcher.worker", 20)
+	v.SetDefault("price_floor.fetcher.capacity", 20000)
 
+	v.SetDefault("account_defaults.events_enabled", false)
 	v.SetDefault("compression.response.enable_gzip", false)
 	v.SetDefault("compression.request.enable_gzip", false)
 

config/config_test.go

@@ -191,6 +191,10 @@ func TestDefaults(t *testing.T) {
 	cmpBools(t, "account_defaults.price_floors.use_dynamic_data", false, cfg.AccountDefaults.PriceFloors.UseDynamicData)
 	cmpInts(t, "account_defaults.price_floors.max_rules", 100, cfg.AccountDefaults.PriceFloors.MaxRule)
 	cmpInts(t, "account_defaults.price_floors.max_schema_dims", 3, cfg.AccountDefaults.PriceFloors.MaxSchemaDims)
+	cmpBools(t, "account_defaults.price_floors.fetch.enabled", false, cfg.AccountDefaults.PriceFloors.Fetch.Enabled)
+	cmpInts(t, "account_defaults.price_floors.fetch.timeout_ms", 100, cfg.AccountDefaults.PriceFloors.Fetch.Timeout)
+	cmpInts(t, "account_defaults.price_floors.fetch.period_sec", 300, cfg.AccountDefaults.PriceFloors.Fetch.Period)
+	cmpInts(t, "account_defaults.price_floors.fetch.max_age_sec", 600, cfg.AccountDefaults.PriceFloors.Fetch.MaxAge)
 	cmpBools(t, "account_defaults.events_enabled", *cfg.AccountDefaults.EventsEnabled, false)
 	cmpNils(t, "account_defaults.events.enabled", cfg.AccountDefaults.Events.Enabled)
 
@@ -465,6 +469,9 @@ hooks:
     enabled: true
 price_floors:
     enabled: true
+    fetcher:
+      worker: 20
+      capacity: 20000
 account_defaults:
     events_enabled: false
     events:
@@ -477,6 +484,11 @@ account_defaults:
         use_dynamic_data: true
         max_rules: 120
         max_schema_dims: 5
+        fetch:
+          enabled: true
+          timeout_ms: 100
+          period_sec: 300
+          max_age_sec: 600
 tmax_adjustments:
   enabled: true
   bidder_response_duration_min_ms: 700
@@ -580,6 +592,11 @@ func TestFullConfig(t *testing.T) {
 	cmpBools(t, "account_defaults.price_floors.use_dynamic_data", true, cfg.AccountDefaults.PriceFloors.UseDynamicData)
 	cmpInts(t, "account_defaults.price_floors.max_rules", 120, cfg.AccountDefaults.PriceFloors.MaxRule)
 	cmpInts(t, "account_defaults.price_floors.max_schema_dims", 5, cfg.AccountDefaults.PriceFloors.MaxSchemaDims)
+	cmpBools(t, "account_defaults.price_floors.fetch.enabled", true, cfg.AccountDefaults.PriceFloors.Fetch.Enabled)
+	cmpInts(t, "account_defaults.price_floors.fetch.timeout_ms", 100, cfg.AccountDefaults.PriceFloors.Fetch.Timeout)
+	cmpInts(t, "account_defaults.price_floors.fetch.period_sec", 300, cfg.AccountDefaults.PriceFloors.Fetch.Period)
+	cmpInts(t, "account_defaults.price_floors.fetch.max_age_sec", 600, cfg.AccountDefaults.PriceFloors.Fetch.MaxAge)
+
 	cmpBools(t, "account_defaults.events_enabled", *cfg.AccountDefaults.EventsEnabled, true)
 	cmpNils(t, "account_defaults.events.enabled", cfg.AccountDefaults.Events.Enabled)
 
@@ -797,6 +814,15 @@ func TestValidateConfig(t *testing.T) {
 			Files:         FileFetcherConfig{Enabled: true},
 			InMemoryCache: InMemoryCache{Type: "none"},
 		},
+		AccountDefaults: Account{
+			PriceFloors: AccountPriceFloors{
+				Fetch: AccountFloorFetch{
+					Timeout: 100,
+					Period:  300,
+					MaxAge:  600,
+				},
+			},
+		},
 	}
 
 	v := viper.New()

endpoints/openrtb2/auction_benchmark_test.go

@@ -95,6 +95,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) {
 		empty_fetcher.EmptyFetcher{},
 		&adscert.NilSigner{},
 		macros.NewStringIndexBasedReplacer(),
+		nil,
 	)
 
 	endpoint, _ := NewEndpoint(

endpoints/openrtb2/test_utils.go

@@ -1213,6 +1213,7 @@ func buildTestExchange(testCfg *testConfigValues, adapterMap map[openrtb_ext.Bid
 		mockFetcher,
 		&adscert.NilSigner{},
 		macros.NewStringIndexBasedReplacer(),
+		nil,
 	)
 
 	testExchange = &exchangeTestWrapper{

exchange/exchange.go

@@ -79,6 +79,7 @@ type exchange struct {
 	requestSplitter          requestSplitter
 	macroReplacer            macros.Replacer
 	floor                    config.PriceFloors
+	priceFloorFetcher        floors.FloorFetcher
 }
 
 // Container to pass out response ext data from the GetAllBids goroutines back into the main thread
@@ -127,7 +128,7 @@ func (randomDeduplicateBidBooleanGenerator) Generate() bool {
 	return rand.Intn(100) < 50
 }
 
-func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, syncersByBidder map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gdprPermsBuilder gdpr.PermissionsBuilder, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher, adsCertSigner adscert.Signer, macroReplacer macros.Replacer) Exchange {
+func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid_cache_client.Client, cfg *config.Configuration, syncersByBidder map[string]usersync.Syncer, metricsEngine metrics.MetricsEngine, infos config.BidderInfos, gdprPermsBuilder gdpr.PermissionsBuilder, currencyConverter *currency.RateConverter, categoriesFetcher stored_requests.CategoryFetcher, adsCertSigner adscert.Signer, macroReplacer macros.Replacer, priceFloorFetcher floors.FloorFetcher) Exchange {
 	bidderToSyncerKey := map[string]string{}
 	for bidder, syncer := range syncersByBidder {
 		bidderToSyncerKey[bidder] = syncer.Key()
@@ -173,6 +174,7 @@ func NewExchange(adapters map[openrtb_ext.BidderName]AdaptedBidder, cache prebid
 		requestSplitter:          requestSplitter,
 		macroReplacer:            macroReplacer,
 		floor:                    cfg.PriceFloors,
+		priceFloorFetcher:        priceFloorFetcher,
 	}
 }
 
@@ -264,7 +266,7 @@ func (e *exchange) HoldAuction(ctx context.Context, r *AuctionRequest, debugLog
 	conversions := e.getAuctionCurrencyRates(requestExtPrebid.CurrencyConversions)
 
 	if e.floor.Enabled {
-		floorErrs = floors.EnrichWithPriceFloors(r.BidRequestWrapper, r.Account, conversions)
+		floorErrs = floors.EnrichWithPriceFloors(r.BidRequestWrapper, r.Account, conversions, e.priceFloorFetcher)
 	}
 
 	responseDebugAllow, accountDebugAllow, debugLog := getDebugInfo(r.BidRequestWrapper.Test, requestExtPrebid, r.Account.DebugAllow, debugLog)