Floors: Dynamic fetch

config/account.go

@@ -58,17 +58,30 @@ type AccountCCPA struct {
 }
 
 type AccountPriceFloors struct {
-	Enabled                bool `mapstructure:"enabled" json:"enabled"`
-	EnforceFloorsRate      int  `mapstructure:"enforce_floors_rate" json:"enforce_floors_rate"`
-	AdjustForBidAdjustment bool `mapstructure:"adjust_for_bid_adjustment" json:"adjust_for_bid_adjustment"`
-	EnforceDealFloors      bool `mapstructure:"enforce_deal_floors" json:"enforce_deal_floors"`
-	UseDynamicData         bool `mapstructure:"use_dynamic_data" json:"use_dynamic_data"`
-	MaxRule                int  `mapstructure:"max_rules" json:"max_rules"`
-	MaxSchemaDims          int  `mapstructure:"max_schema_dims" json:"max_schema_dims"`
+	Enabled                bool              `mapstructure:"enabled" json:"enabled"`
+	EnforceFloorsRate      int               `mapstructure:"enforce_floors_rate" json:"enforce_floors_rate"`
+	AdjustForBidAdjustment bool              `mapstructure:"adjust_for_bid_adjustment" json:"adjust_for_bid_adjustment"`
+	EnforceDealFloors      bool              `mapstructure:"enforce_deal_floors" json:"enforce_deal_floors"`
+	UseDynamicData         bool              `mapstructure:"use_dynamic_data" json:"use_dynamic_data"`
+	MaxRule                int               `mapstructure:"max_rules" json:"max_rules"`
+	MaxSchemaDims          int               `mapstructure:"max_schema_dims" json:"max_schema_dims"`
+	Fetcher                AccountFloorFetch `mapstructure:"fetch" json:"fetch"`
 }
 
-func (pf *AccountPriceFloors) validate(errs []error) []error {
+// AccountFloorFetch defines the configuration for dynamic floors fetching.
+type AccountFloorFetch struct {
+	Enabled       bool   `mapstructure:"enabled" json:"enabled"`
+	URL           string `mapstructure:"url" json:"url"`
+	Timeout       int    `mapstructure:"timeout_ms" json:"timeout_ms"`
+	MaxFileSizeKB int    `mapstructure:"max_file_size_kb" json:"max_file_size_kb"`
+	MaxRules      int    `mapstructure:"max_rules" json:"max_rules"`
+	MaxAge        int    `mapstructure:"max_age_sec" json:"max_age_sec"`
+	Period        int    `mapstructure:"period_sec" json:"period_sec"`
+	MaxSchemaDims int    `mapstructure:"max_schema_dims" json:"max_schema_dims"`
+	AccountID     string `mapstructure:"accountID" json:"accountID"`
+}
 
+func (pf *AccountPriceFloors) validate(errs []error) []error {
 	if pf.EnforceFloorsRate < 0 || pf.EnforceFloorsRate > 100 {
 		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.enforce_floors_rate should be between 0 and 100`))
 	}
@@ -81,6 +94,34 @@ func (pf *AccountPriceFloors) validate(errs []error) []error {
 		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.max_schema_dims should be between 0 and 20`))
 	}
 
+	if pf.Fetcher.Period > pf.Fetcher.MaxAge {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.period_sec should be less than account_defaults.price_floors.fetch.max_age_sec`))
+	}
+
+	if pf.Fetcher.Period < 300 {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.period_sec should not be less than 300 seconds`))
+	}
+
+	if pf.Fetcher.MaxAge < 600 {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_age_sec should not be less than 600 seconds and greater than maximum integer value`))
+	}
+
+	if !(pf.Fetcher.Timeout > 10 && pf.Fetcher.Timeout < 10000) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.timeout_ms should be between 10 to 10,000 miliseconds`))
+	}
+
+	if pf.Fetcher.MaxRules < 0 {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_rules should be greater than or equal to 0`))
+	}
+
+	if pf.Fetcher.MaxFileSizeKB < 0 {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_file_size_kb should be greater than or equal to 0`))
+	}
+
+	if !(pf.Fetcher.MaxSchemaDims >= 0 && pf.Fetcher.MaxSchemaDims < 20) {
+		errs = append(errs, fmt.Errorf(`account_defaults.price_floors.fetch.max_schema_dims should not be less than 0 and greater than 20`))
+	}
+
 	return errs
 }
 

config/account_test.go

@@ -795,7 +795,6 @@ func TestModulesGetConfig(t *testing.T) {
 }
 
 func TestAccountPriceFloorsValidate(t *testing.T) {
-
 	tests := []struct {
 		description string
 		pf          *AccountPriceFloors
@@ -807,36 +806,163 @@ func TestAccountPriceFloorsValidate(t *testing.T) {
 				EnforceFloorsRate: 100,
 				MaxRule:           200,
 				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 		},
 		{
 			description: "Invalid configuration: EnforceFloorRate:110",
 			pf: &AccountPriceFloors{
 				EnforceFloorsRate: 110,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.enforce_floors_rate should be between 0 and 100")},
 		},
 		{
 			description: "Invalid configuration: EnforceFloorRate:-10",
 			pf: &AccountPriceFloors{
 				EnforceFloorsRate: -10,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.enforce_floors_rate should be between 0 and 100")},
 		},
 		{
 			description: "Invalid configuration: MaxRule:-20",
 			pf: &AccountPriceFloors{
 				MaxRule: -20,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.max_rules should be between 0 and 2147483647")},
 		},
 		{
 			description: "Invalid configuration: MaxSchemaDims:100",
 			pf: &AccountPriceFloors{
 				MaxSchemaDims: 100,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 12,
+				},
 			},
 			want: []error{errors.New("account_defaults.price_floors.max_schema_dims should be between 0 and 20")},
 		},
+		{
+			description: "Invalid period for fetch",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:  100,
+					MaxAge:  600,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.period_sec should not be less than 300 seconds")},
+		},
+		{
+			description: "Invalid max age for fetch",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  500,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_age_sec should not be less than 600 seconds and greater than maximum integer value")},
+		},
+		{
+			description: "Period is greater than max age",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:  700,
+					MaxAge:  600,
+					Timeout: 12,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.period_sec should be less than account_defaults.price_floors.fetch.max_age_sec")},
+		},
+		{
+			description: "Invalid timeout",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:  300,
+					MaxAge:  600,
+					Timeout: 4,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.timeout_ms should be between 10 to 10,000 miliseconds")},
+		},
+		{
+			description: "Invalid Max Rules",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:   300,
+					MaxAge:   600,
+					Timeout:  12,
+					MaxRules: -2,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_rules should be greater than or equal to 0")},
+		},
+		{
+			description: "Invalid Max File size",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:        300,
+					MaxAge:        600,
+					Timeout:       12,
+					MaxFileSizeKB: -1,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_file_size_kb should be greater than or equal to 0")},
+		},
+		{
+			description: "Invalid max_schema_dims",
+			pf: &AccountPriceFloors{
+				EnforceFloorsRate: 100,
+				MaxRule:           200,
+				MaxSchemaDims:     10,
+				Fetcher: AccountFloorFetch{
+					Period:        300,
+					MaxAge:        600,
+					Timeout:       12,
+					MaxFileSizeKB: 10,
+					MaxSchemaDims: 40,
+				},
+			},
+			want: []error{errors.New("account_defaults.price_floors.fetch.max_schema_dims should not be less than 0 and greater than 20")},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.description, func(t *testing.T) {

config/config.go

@@ -102,7 +102,16 @@ type Configuration struct {
 }
 
 type PriceFloors struct {
-	Enabled bool `mapstructure:"enabled"`
+	Enabled bool              `mapstructure:"enabled"`
+	Fetcher PriceFloorFetcher `mapstructure:"fetcher"`
+}
+
+type PriceFloorFetcher struct {
+	HttpClient HTTPClient `mapstructure:"http_client"`
+	CacheSize  int        `mapstructure:"cache_size_mb"`
+	Worker     int        `mapstructure:"worker"`
+	Capacity   int        `mapstructure:"capacity"`
+	MaxRetries int        `mapstructure:"max_retries"`
 }
 
 const MIN_COOKIE_SIZE_BYTES = 500
@@ -139,10 +148,6 @@ func (cfg *Configuration) validate(v *viper.Viper) []error {
 		glog.Warning(`account_defaults.events has no effect as the feature is under development.`)
 	}
 
-	if cfg.PriceFloors.Enabled {
-		glog.Warning(`cfg.PriceFloors.Enabled will currently not do anything as price floors feature is still under development.`)
-	}
-
 	errs = cfg.Experiment.validate(errs)
 	errs = cfg.BidderInfos.validate(errs)
 	errs = cfg.AccountDefaults.Privacy.IPv6Config.Validate(errs)
@@ -1059,9 +1064,30 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
 	v.SetDefault("account_defaults.price_floors.use_dynamic_data", false)
 	v.SetDefault("account_defaults.price_floors.max_rules", 100)
 	v.SetDefault("account_defaults.price_floors.max_schema_dims", 3)
+	v.SetDefault("account_defaults.price_floors.fetch.enabled", false)
+	v.SetDefault("account_defaults.price_floors.fetch.url", "")
+	v.SetDefault("account_defaults.price_floors.fetch.timeout_ms", 3000)
+	v.SetDefault("account_defaults.price_floors.fetch.max_file_size_kb", 100)
+	v.SetDefault("account_defaults.price_floors.fetch.max_rules", 1000)
+	v.SetDefault("account_defaults.price_floors.fetch.max_age_sec", 86400)
+	v.SetDefault("account_defaults.price_floors.fetch.period_sec", 3600)
+	v.SetDefault("account_defaults.price_floors.fetch.max_schema_dims", 0)
+
+	v.SetDefault("account_defaults.events_enabled", false)
 	v.SetDefault("account_defaults.privacy.ipv6.anon_keep_bits", 56)
 	v.SetDefault("account_defaults.privacy.ipv4.anon_keep_bits", 24)
 
+	//Defaults for Price floor fetcher
+	v.SetDefault("price_floors.fetcher.worker", 20)
+	v.SetDefault("price_floors.fetcher.capacity", 20000)
+	v.SetDefault("price_floors.fetcher.cache_size_mb", 64)
+	v.SetDefault("price_floors.fetcher.http_client.max_connections_per_host", 0) // unlimited
+	v.SetDefault("price_floors.fetcher.http_client.max_idle_connections", 40)
+	v.SetDefault("price_floors.fetcher.http_client.max_idle_connections_per_host", 2)
+	v.SetDefault("price_floors.fetcher.http_client.idle_connection_timeout_seconds", 60)
+	v.SetDefault("price_floors.fetcher.max_retries", 10)
+
+	v.SetDefault("account_defaults.events_enabled", false)
 	v.SetDefault("compression.response.enable_gzip", false)
 	v.SetDefault("compression.request.enable_gzip", false)