diff --git a/config/config.go b/config/config.go index e564c34ae3a..f503d0fcb59 100644 --- a/config/config.go +++ b/config/config.go @@ -944,6 +944,8 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) { v.SetDefault("event.timeout_ms", 1000) + v.SetDefault("user_sync.priority_groups", [][]string{}) + v.SetDefault("accounts.filesystem.enabled", false) v.SetDefault("accounts.filesystem.directorypath", "./stored_requests/data/by_id") v.SetDefault("accounts.in_memory_cache.type", "none") diff --git a/config/usersync.go b/config/usersync.go index 27badbcb815..d490853d08e 100644 --- a/config/usersync.go +++ b/config/usersync.go @@ -2,13 +2,13 @@ package config // UserSync specifies the static global user sync configuration. type UserSync struct { - Cooperative UserSyncCooperative `mapstructure:"coop_sync"` - ExternalURL string `mapstructure:"external_url"` - RedirectURL string `mapstructure:"redirect_url"` + Cooperative UserSyncCooperative `mapstructure:"coop_sync"` + ExternalURL string `mapstructure:"external_url"` + RedirectURL string `mapstructure:"redirect_url"` + PriorityGroups [][]string `mapstructure:"priority_groups"` } // UserSyncCooperative specifies the static global default cooperative cookie sync type UserSyncCooperative struct { - EnabledByDefault bool `mapstructure:"default"` - PriorityGroups [][]string `mapstructure:"priority_groups"` + EnabledByDefault bool `mapstructure:"default"` } diff --git a/endpoints/cookie_sync.go b/endpoints/cookie_sync.go index 0a81c2d5e9c..250d5fd07d3 100644 --- a/endpoints/cookie_sync.go +++ b/endpoints/cookie_sync.go @@ -39,6 +39,7 @@ var ( errCookieSyncAccountBlocked = errors.New("account is disabled, please reach out to the prebid server host") errCookieSyncAccountConfigMalformed = errors.New("account config is malformed and could not be read") errCookieSyncAccountInvalid = errors.New("account must be valid if provided, please reach out to the prebid server host") + errSyncerIsNotPriority = errors.New("syncer key is not a priority, and there are only priority elements left") ) var cookieSyncBidderFilterAllowAll = usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude) @@ -168,7 +169,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, ma Bidders: request.Bidders, Cooperative: usersync.Cooperative{ Enabled: (request.CooperativeSync != nil && *request.CooperativeSync) || (request.CooperativeSync == nil && c.config.UserSync.Cooperative.EnabledByDefault), - PriorityGroups: c.config.UserSync.Cooperative.PriorityGroups, + PriorityGroups: c.config.UserSync.PriorityGroups, }, Limit: request.Limit, Privacy: usersyncPrivacy{ diff --git a/endpoints/cookie_sync_test.go b/endpoints/cookie_sync_test.go index 68ab0ced44e..29fe74b273c 100644 --- a/endpoints/cookie_sync_test.go +++ b/endpoints/cookie_sync_test.go @@ -515,9 +515,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{ @@ -557,9 +557,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{ @@ -608,9 +608,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: true, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -635,9 +635,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -662,9 +662,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: true, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -689,9 +689,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -716,9 +716,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: true, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -743,9 +743,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -890,9 +890,8 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ - Cooperative: config.UserSyncCooperative{ - PriorityGroups: [][]string{{"a", "b", "c"}}, - }, + PriorityGroups: [][]string{{"a", "b", "c"}}, + Cooperative: config.UserSyncCooperative{}, }, expectedPrivacy: macros.UserSyncPrivacy{}, expectedRequest: usersync.Request{ @@ -921,9 +920,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, @@ -953,9 +952,9 @@ func TestCookieSyncParseRequest(t *testing.T) { givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"}, givenCCPAEnabled: true, givenConfig: config.UserSync{ + PriorityGroups: [][]string{{"a", "b", "c"}}, Cooperative: config.UserSyncCooperative{ EnabledByDefault: false, - PriorityGroups: [][]string{{"a", "b", "c"}}, }, }, expectedPrivacy: macros.UserSyncPrivacy{}, diff --git a/endpoints/setuid.go b/endpoints/setuid.go index 83425789647..520a133b51e 100644 --- a/endpoints/setuid.go +++ b/endpoints/setuid.go @@ -158,11 +158,22 @@ func NewSetUIDEndpoint(cfg *config.Configuration, syncersByBidder map[string]use setSiteCookie := siteCookieCheck(r.UserAgent()) + // Priority Ejector Set Up + priorityEjector := &usersync.PriorityBidderEjector{PriorityGroups: cfg.UserSync.PriorityGroups, TieEjector: &usersync.OldestEjector{}, SyncersByBidder: syncersByBidder} + priorityEjector.IsSyncerPriority = isSyncerPriority(bidderName, cfg.UserSync.PriorityGroups) + // Write Cookie - encodedCookie, err := cookie.PrepareCookieForWrite(&cfg.HostCookie, encoder) + encodedCookie, err := cookie.PrepareCookieForWrite(&cfg.HostCookie, encoder, priorityEjector) if err != nil { - handleBadStatus(w, http.StatusBadRequest, metrics.SetUidBadRequest, err, metricsEngine, &so) - return + if err.Error() == errSyncerIsNotPriority.Error() { + w.WriteHeader(http.StatusOK) + w.Write([]byte("Warning: " + err.Error() + ", cookie not updated")) + so.Status = http.StatusOK + return + } else { + handleBadStatus(w, http.StatusBadRequest, metrics.SetUidBadRequest, err, metricsEngine, &so) + return + } } usersync.WriteCookie(w, encodedCookie, &cfg.HostCookie, setSiteCookie) @@ -326,6 +337,17 @@ func getSyncer(query url.Values, syncersByBidder map[string]usersync.Syncer) (us return syncer, bidder, nil } +func isSyncerPriority(bidderNameFromSyncerQuery string, priorityGroups [][]string) bool { + for _, group := range priorityGroups { + for _, bidder := range group { + if bidderNameFromSyncerQuery == bidder { + return true + } + } + } + return false +} + // getResponseFormat reads the format query parameter or falls back to the syncer's default. // Returns either "b" (iframe), "i" (redirect), or an empty string "" (legacy behavior of an // empty response body with no content type). diff --git a/endpoints/setuid_test.go b/endpoints/setuid_test.go index 4f37eb77f8f..897538c35e5 100644 --- a/endpoints/setuid_test.go +++ b/endpoints/setuid_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "net/url" "regexp" + "strings" "testing" "time" @@ -332,7 +333,7 @@ func TestSetUIDEndpoint(t *testing.T) { for _, test := range testCases { response := doRequest(makeRequest(test.uri, test.existingSyncs), analytics, metrics, - test.syncersBidderNameToKey, test.gdprAllowsHostCookies, test.gdprReturnsError, test.gdprMalformed, false) + test.syncersBidderNameToKey, test.gdprAllowsHostCookies, test.gdprReturnsError, test.gdprMalformed, false, 0, nil) assert.Equal(t, test.expectedStatusCode, response.Code, "Test Case: %s. /setuid returned unexpected error code", test.description) if test.expectedSyncs != nil { @@ -359,6 +360,135 @@ func TestSetUIDEndpoint(t *testing.T) { } } +func TestSetUIDPriorityEjection(t *testing.T) { + decoder := usersync.Base64Decoder{} + analytics := analyticsConf.NewPBSAnalytics(&config.Analytics{}) + syncersByBidder := map[string]string{ + "pubmatic": "pubmatic", + "syncer1": "syncer1", + "syncer2": "syncer2", + "syncer3": "syncer3", + "syncer4": "syncer4", + "mismatchedBidderName": "syncer5", + "syncerToEject": "syncerToEject", + } + + testCases := []struct { + description string + uri string + givenExistingSyncs []string + givenPriorityGroups [][]string + givenMaxCookieSize int + expectedStatusCode int + expectedSyncer string + expectedUID string + expectedNumOfElements int + expectedWarning string + }{ + { + description: "Cookie empty, expect bidder to be synced, no ejection", + uri: "/setuid?bidder=pubmatic&uid=123", + givenPriorityGroups: [][]string{}, + givenMaxCookieSize: 500, + expectedSyncer: "pubmatic", + expectedUID: "123", + expectedNumOfElements: 1, + expectedStatusCode: http.StatusOK, + }, + { + description: "Cookie full, no priority groups, one ejection", + uri: "/setuid?bidder=pubmatic&uid=123", + givenExistingSyncs: []string{"syncer1", "syncer2", "syncer3", "syncer4"}, + givenPriorityGroups: [][]string{}, + givenMaxCookieSize: 500, + expectedUID: "123", + expectedSyncer: "pubmatic", + expectedNumOfElements: 4, + expectedStatusCode: http.StatusOK, + }, + { + description: "Cookie full, eject lowest priority element", + uri: "/setuid?bidder=pubmatic&uid=123", + givenExistingSyncs: []string{"syncer2", "syncer3", "syncer4", "syncerToEject"}, + givenPriorityGroups: [][]string{{"pubmatic", "syncer2", "syncer3", "syncer4"}, {"syncerToEject"}}, + givenMaxCookieSize: 500, + expectedUID: "123", + expectedSyncer: "pubmatic", + expectedNumOfElements: 4, + expectedStatusCode: http.StatusOK, + }, + { + description: "Cookie full, all elements same priority, one ejection", + uri: "/setuid?bidder=pubmatic&uid=123", + givenExistingSyncs: []string{"syncer1", "syncer2", "syncer3", "syncer5"}, + givenPriorityGroups: [][]string{{"pubmatic", "syncer1", "syncer2", "syncer3", "mismatchedBidderName"}}, + givenMaxCookieSize: 500, + expectedUID: "123", + expectedSyncer: "pubmatic", + expectedNumOfElements: 4, + expectedStatusCode: http.StatusOK, + }, + { + description: "There are only priority elements left, but the bidder being synced isn't one", + uri: "/setuid?bidder=pubmatic&uid=123", + givenExistingSyncs: []string{"syncer1", "syncer2", "syncer3", "syncer4"}, + givenPriorityGroups: [][]string{{"syncer1", "syncer2", "syncer3", "syncer4"}}, + givenMaxCookieSize: 500, + expectedStatusCode: http.StatusOK, + expectedWarning: "Warning: syncer key is not a priority, and there are only priority elements left, cookie not updated", + }, + { + description: "Uid that's trying to be synced is bigger than MaxCookieSize", + uri: "/setuid?bidder=pubmatic&uid=123", + givenMaxCookieSize: 1, + expectedStatusCode: http.StatusBadRequest, + }, + } + for _, test := range testCases { + request := httptest.NewRequest("GET", test.uri, nil) + + // Cookie Set Up + cookie := usersync.NewCookie() + for _, key := range test.givenExistingSyncs { + cookie.Sync(key, "111") + } + httpCookie, err := ToHTTPCookie(cookie) + assert.NoError(t, err) + request.AddCookie(httpCookie) + + // Make Request to /setuid + response := doRequest(request, analytics, &metricsConf.NilMetricsEngine{}, syncersByBidder, true, false, false, false, test.givenMaxCookieSize, test.givenPriorityGroups) + + if test.expectedWarning != "" { + assert.Equal(t, test.expectedWarning, response.Body.String(), test.description) + } else if test.expectedSyncer != "" { + // Get Cookie From Header + var cookieHeader string + for k, v := range response.Result().Header { + if k == "Set-Cookie" { + cookieHeader = v[0] + } + } + encodedCookieValue := getUIDFromHeader(cookieHeader) + + // Check That Bidder On Request was Synced, it's UID matches, and that the right number of elements are present after ejection + decodedCookie := decoder.Decode(encodedCookieValue) + decodedCookieUIDs := decodedCookie.GetUIDs() + + assert.Equal(t, test.expectedUID, decodedCookieUIDs[test.expectedSyncer], test.description) + assert.Equal(t, test.expectedNumOfElements, len(decodedCookieUIDs), test.description) + + // Specific test case handling where we eject the lowest priority element + if len(test.givenPriorityGroups) == 2 { + syncer := test.givenPriorityGroups[len(test.givenPriorityGroups)-1][0] + _, syncerExists := decodedCookieUIDs[syncer] + assert.False(t, syncerExists, test.description) + } + } + assert.Equal(t, test.expectedStatusCode, response.Result().StatusCode, test.description) + } +} + func TestParseSignalFromGPPSID(t *testing.T) { type testOutput struct { signal gdpr.Signal @@ -1214,7 +1344,7 @@ func TestSetUIDEndpointMetrics(t *testing.T) { for _, v := range test.cookies { addCookie(req, v) } - response := doRequest(req, analyticsEngine, metricsEngine, test.syncersBidderNameToKey, test.gdprAllowsHostCookies, false, false, test.cfgAccountRequired) + response := doRequest(req, analyticsEngine, metricsEngine, test.syncersBidderNameToKey, test.gdprAllowsHostCookies, false, false, test.cfgAccountRequired, 0, nil) assert.Equal(t, test.expectedResponseCode, response.Code, test.description) analyticsEngine.AssertExpectations(t) @@ -1230,7 +1360,7 @@ func TestOptedOut(t *testing.T) { syncersBidderNameToKey := map[string]string{"pubmatic": "pubmatic"} analytics := analyticsConf.NewPBSAnalytics(&config.Analytics{}) metrics := &metricsConf.NilMetricsEngine{} - response := doRequest(request, analytics, metrics, syncersBidderNameToKey, true, false, false, false) + response := doRequest(request, analytics, metrics, syncersBidderNameToKey, true, false, false, false, 0, nil) assert.Equal(t, http.StatusUnauthorized, response.Code) } @@ -1341,6 +1471,56 @@ func TestGetResponseFormat(t *testing.T) { } } +func TestIsSyncerPriority(t *testing.T) { + testCases := []struct { + name string + givenBidderNameFromSyncerQuery string + givenPriorityGroups [][]string + expected bool + }{ + { + name: "bidder-name-is-priority", + givenBidderNameFromSyncerQuery: "priorityBidder", + givenPriorityGroups: [][]string{ + {"priorityBidder"}, + {"2", "3"}, + }, + expected: true, + }, + { + name: "bidder-name-is-not-priority", + givenBidderNameFromSyncerQuery: "notPriorityBidderName", + givenPriorityGroups: [][]string{ + {"1"}, + {"2", "3"}, + }, + expected: false, + }, + { + name: "no-bidder-name-given", + givenBidderNameFromSyncerQuery: "", + givenPriorityGroups: [][]string{ + {"1"}, + {"2", "3"}, + }, + expected: false, + }, + { + name: "no-priority-groups-given", + givenBidderNameFromSyncerQuery: "bidderName", + givenPriorityGroups: [][]string{}, + expected: false, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + isPriority := isSyncerPriority(test.givenBidderNameFromSyncerQuery, test.givenPriorityGroups) + assert.Equal(t, test.expected, isPriority) + }) + } +} + func assertHasSyncs(t *testing.T, testCase string, resp *httptest.ResponseRecorder, syncs map[string]string) { t.Helper() cookie := parseCookieString(t, resp) @@ -1366,13 +1546,19 @@ func makeRequest(uri string, existingSyncs map[string]string) *http.Request { return request } -func doRequest(req *http.Request, analytics analytics.PBSAnalyticsModule, metrics metrics.MetricsEngine, syncersBidderNameToKey map[string]string, gdprAllowsHostCookies, gdprReturnsError, gdprReturnsMalformedError, cfgAccountRequired bool) *httptest.ResponseRecorder { +func doRequest(req *http.Request, analytics analytics.PBSAnalyticsModule, metrics metrics.MetricsEngine, syncersBidderNameToKey map[string]string, gdprAllowsHostCookies, gdprReturnsError, gdprReturnsMalformedError, cfgAccountRequired bool, maxCookieSize int, priorityGroups [][]string) *httptest.ResponseRecorder { cfg := config.Configuration{ AccountRequired: cfgAccountRequired, BlacklistedAcctMap: map[string]bool{ "blocked_acct": true, }, AccountDefaults: config.Account{}, + UserSync: config.UserSync{ + PriorityGroups: priorityGroups, + }, + HostCookie: config.HostCookie{ + MaxCookieSizeBytes: maxCookieSize, + }, } cfg.MarshalAccountDefaults() @@ -1395,6 +1581,10 @@ func doRequest(req *http.Request, analytics analytics.PBSAnalyticsModule, metric syncersByBidder := make(map[string]usersync.Syncer) for bidderName, syncerKey := range syncersBidderNameToKey { syncersByBidder[bidderName] = fakeSyncer{key: syncerKey, defaultSyncType: usersync.SyncTypeIFrame} + if priorityGroups == nil { + cfg.UserSync.PriorityGroups = [][]string{{}} + cfg.UserSync.PriorityGroups[0] = append(cfg.UserSync.PriorityGroups[0], bidderName) + } } fakeAccountsFetcher := FakeAccountsFetcher{AccountData: map[string]json.RawMessage{ @@ -1513,3 +1703,17 @@ func ToHTTPCookie(cookie *usersync.Cookie) (*http.Cookie, error) { Path: "/", }, nil } + +func getUIDFromHeader(setCookieHeader string) string { + cookies := strings.Split(setCookieHeader, ";") + for _, cookie := range cookies { + trimmedCookie := strings.TrimSpace(cookie) + if strings.HasPrefix(trimmedCookie, "uids=") { + parts := strings.SplitN(trimmedCookie, "=", 2) + if len(parts) == 2 { + return parts[1] + } + } + } + return "" +} diff --git a/pbs/usersync.go b/pbs/usersync.go index 7b468cb039d..a5b49f6db03 100644 --- a/pbs/usersync.go +++ b/pbs/usersync.go @@ -22,6 +22,7 @@ type UserSyncDeps struct { ExternalUrl string RecaptchaSecret string HostCookieConfig *config.HostCookie + PriorityGroups [][]string } // Struct for parsing json in google's response @@ -81,7 +82,7 @@ func (deps *UserSyncDeps) OptOut(w http.ResponseWriter, r *http.Request, _ httpr pc.SetOptOut(optout != "") // Write Cookie - encodedCookie, err := pc.PrepareCookieForWrite(deps.HostCookieConfig, encoder) + encodedCookie, err := encoder.Encode(pc) if err != nil { w.WriteHeader(http.StatusBadRequest) return diff --git a/router/router.go b/router/router.go index f7270445ff4..f2b1f7e7447 100644 --- a/router/router.go +++ b/router/router.go @@ -274,6 +274,7 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R HostCookieConfig: &(cfg.HostCookie), ExternalUrl: cfg.ExternalURL, RecaptchaSecret: cfg.RecaptchaSecret, + PriorityGroups: cfg.UserSync.PriorityGroups, } r.GET("/setuid", endpoints.NewSetUIDEndpoint(cfg, syncersByBidder, gdprPermsBuilder, tcf2CfgBuilder, pbsAnalytics, accounts, r.MetricsEngine)) diff --git a/usersync/cookie.go b/usersync/cookie.go index c0eb898c5ea..94ada94ed75 100644 --- a/usersync/cookie.go +++ b/usersync/cookie.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "net/http" - "sort" "time" "github.com/prebid/prebid-server/config" @@ -58,10 +57,7 @@ func ReadCookie(r *http.Request, decoder Decoder, host *config.HostCookie) *Cook } // PrepareCookieForWrite ejects UIDs as long as the cookie is too full -func (cookie *Cookie) PrepareCookieForWrite(cfg *config.HostCookie, encoder Encoder) (string, error) { - uuidKeys := sortUIDs(cookie.uids) - - i := 0 +func (cookie *Cookie) PrepareCookieForWrite(cfg *config.HostCookie, encoder Encoder, ejector Ejector) (string, error) { for len(cookie.uids) > 0 { encodedCookie, err := encoder.Encode(cookie) if err != nil { @@ -80,12 +76,15 @@ func (cookie *Cookie) PrepareCookieForWrite(cfg *config.HostCookie, encoder Enco isCookieTooBig := cookieSize > cfg.MaxCookieSizeBytes && cfg.MaxCookieSizeBytes > 0 if !isCookieTooBig { return encodedCookie, nil + } else if len(cookie.uids) == 1 { + return "", errors.New("uid that's trying to be synced is bigger than MaxCookieSize") } - uidToDelete := uuidKeys[i] + uidToDelete, err := ejector.Choose(cookie.uids) + if err != nil { + return encodedCookie, err + } delete(cookie.uids, uidToDelete) - - i++ } return "", nil } @@ -132,23 +131,6 @@ func (cookie *Cookie) Sync(key string, uid string) error { return nil } -// sortUIDs is used to get a list of uids sorted from oldest to newest -// This list is used to eject oldest uids from the cookie -// This will be incorporated with a more complex ejection framework in a future PR -func sortUIDs(uids map[string]UIDEntry) []string { - if len(uids) > 0 { - uuidKeys := make([]string, 0, len(uids)) - for key := range uids { - uuidKeys = append(uuidKeys, key) - } - sort.SliceStable(uuidKeys, func(i, j int) bool { - return uids[uuidKeys[i]].Expires.Before(uids[uuidKeys[j]].Expires) - }) - return uuidKeys - } - return nil -} - // SyncHostCookie syncs the request cookie with the host cookie func SyncHostCookie(r *http.Request, requestCookie *Cookie, host *config.HostCookie) { if uid, _, _ := requestCookie.GetUID(host.Family); uid == "" && host.CookieName != "" { diff --git a/usersync/cookie_test.go b/usersync/cookie_test.go index 5b86df54441..340069767b1 100644 --- a/usersync/cookie_test.go +++ b/usersync/cookie_test.go @@ -380,82 +380,155 @@ func TestWriteCookieUserAgent(t *testing.T) { func TestPrepareCookieForWrite(t *testing.T) { encoder := Base64Encoder{} decoder := Base64Decoder{} - cookieToSend := &Cookie{ + + mainCookie := &Cookie{ + uids: map[string]UIDEntry{ + "mainUID": newTempId("1234567890123456789012345678901234567890123456", 7), + "2": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 6), + "3": newTempId("123456789012345678901234567896123456789012345678", 5), + "4": newTempId("aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ", 4), + "5": newTempId("12345678901234567890123456789012345678901234567890", 3), + "6": newTempId("abcdefghij", 2), + "7": newTempId("abcdefghijklmnopqrstuvwxy", 1), + }, + optOut: false, + } + + errorCookie := &Cookie{ uids: map[string]UIDEntry{ - "1": newTempId("1234567890123456789012345678901234567890123456", 7), - "7": newTempId("abcdefghijklmnopqrstuvwxy", 1), - "2": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 6), - "3": newTempId("123456789012345678901234567896123456789012345678", 5), - "4": newTempId("aAbBcCdDeEfFgGhHiIjJkKlLmMnNoOpPqQrRsStTuUvVwWxXyYzZ", 4), - "5": newTempId("12345678901234567890123456789012345678901234567890", 3), - "6": newTempId("abcdefghij", 2), + "syncerNotPriority": newTempId("ABCDEFGHIJKLMNOPQRSTUVWXYZ", 7), + "2": newTempId("1234567890123456789012345678901234567890123456", 7), // Priority Element }, optOut: false, } + ejector := &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"mainUID"}, + {"2", "3"}, + {"4", "5", "6"}, + {"7"}, + }, + SyncersByBidder: map[string]Syncer{ + "mainUID": fakeSyncer{ + key: "mainUID", + }, + "2": fakeSyncer{ + key: "2", + }, + "3": fakeSyncer{ + key: "3", + }, + "4": fakeSyncer{ + key: "4", + }, + "5": fakeSyncer{ + key: "5", + }, + "6": fakeSyncer{ + key: "6", + }, + "mistmatchedBidder": fakeSyncer{ + key: "7", + }, + }, + TieEjector: &OldestEjector{}, + } + testCases := []struct { name string givenMaxCookieSize int + givenCookieToSend *Cookie + givenIsSyncerPriority bool expectedRemainingUidKeys []string + expectedError error }{ { - name: "no-uids-ejected", - givenMaxCookieSize: 2000, + name: "no-uids-ejected", + givenMaxCookieSize: 2000, + givenCookieToSend: mainCookie, + givenIsSyncerPriority: true, expectedRemainingUidKeys: []string{ - "1", "2", "3", "4", "5", "6", "7", + "mainUID", "2", "3", "4", "5", "6", "7", }, }, { - name: "no-uids-ejected-2", - givenMaxCookieSize: 0, + name: "invalid-max-size", + givenMaxCookieSize: -100, + givenCookieToSend: mainCookie, expectedRemainingUidKeys: []string{ - "1", "2", "3", "4", "5", "6", "7", + "mainUID", "2", "3", "4", "5", "6", "7", }, }, { - name: "one-uid-ejected", - givenMaxCookieSize: 900, + name: "syncer-is-not-priority", + givenMaxCookieSize: 100, + givenCookieToSend: errorCookie, + givenIsSyncerPriority: false, + expectedError: errors.New("syncer key is not a priority, and there are only priority elements left"), + }, + { + name: "no-uids-ejected-2", + givenMaxCookieSize: 0, + givenCookieToSend: mainCookie, + givenIsSyncerPriority: true, expectedRemainingUidKeys: []string{ - "1", "2", "3", "4", "5", "6", + "mainUID", "2", "3", "4", "5", "6", "7", }, }, { - name: "four-uids-ejected", - givenMaxCookieSize: 500, + name: "one-uid-ejected", + givenMaxCookieSize: 900, + givenCookieToSend: mainCookie, + givenIsSyncerPriority: true, expectedRemainingUidKeys: []string{ - "1", "2", "3", + "mainUID", "2", "3", "4", "5", "6", }, }, { - name: "all-but-one-uids-ejected", - givenMaxCookieSize: 300, + name: "four-uids-ejected", + givenMaxCookieSize: 500, + givenCookieToSend: mainCookie, + givenIsSyncerPriority: true, expectedRemainingUidKeys: []string{ - "1", + "mainUID", "2", "3", }, }, { - name: "all-uids-ejected", - givenMaxCookieSize: 100, - expectedRemainingUidKeys: []string{}, + name: "all-but-one-uids-ejected", + givenMaxCookieSize: 300, + givenCookieToSend: mainCookie, + givenIsSyncerPriority: true, + expectedRemainingUidKeys: []string{ + "mainUID", + }, }, { - name: "invalid-max-size", - givenMaxCookieSize: -100, + name: "only-main-uid-left", + givenMaxCookieSize: 100, + givenCookieToSend: mainCookie, + expectedError: errors.New("uid that's trying to be synced is bigger than MaxCookieSize"), expectedRemainingUidKeys: []string{}, }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { - encodedCookie, err := cookieToSend.PrepareCookieForWrite(&config.HostCookie{MaxCookieSizeBytes: test.givenMaxCookieSize}, encoder) - assert.NoError(t, err) - decodedCookie := decoder.Decode(encodedCookie) + ejector.IsSyncerPriority = test.givenIsSyncerPriority + encodedCookie, err := test.givenCookieToSend.PrepareCookieForWrite(&config.HostCookie{MaxCookieSizeBytes: test.givenMaxCookieSize}, encoder, ejector) + + if test.expectedError != nil { + assert.Equal(t, test.expectedError, err) + } else { + assert.NoError(t, err) + decodedCookie := decoder.Decode(encodedCookie) - for _, key := range test.expectedRemainingUidKeys { - _, ok := decodedCookie.uids[key] - assert.Equal(t, true, ok) + for _, key := range test.expectedRemainingUidKeys { + _, ok := decodedCookie.uids[key] + assert.Equal(t, true, ok) + } + assert.Equal(t, len(decodedCookie.uids), len(test.expectedRemainingUidKeys)) } - assert.Equal(t, len(decodedCookie.uids), len(test.expectedRemainingUidKeys)) }) } } diff --git a/usersync/ejector.go b/usersync/ejector.go new file mode 100644 index 00000000000..84299f72c12 --- /dev/null +++ b/usersync/ejector.go @@ -0,0 +1,127 @@ +package usersync + +import ( + "errors" + "time" +) + +type Ejector interface { + Choose(uids map[string]UIDEntry) (string, error) +} + +type OldestEjector struct{} + +type PriorityBidderEjector struct { + PriorityGroups [][]string + SyncersByBidder map[string]Syncer + IsSyncerPriority bool + TieEjector Ejector +} + +// Choose method for oldest ejector will return the oldest uid +func (o *OldestEjector) Choose(uids map[string]UIDEntry) (string, error) { + var oldestElement string + var oldestDate time.Time = time.Unix(1<<63-62135596801, 999999999) // Max value for time + + for key, value := range uids { + if value.Expires.Before(oldestDate) { + oldestElement = key + oldestDate = value.Expires + } + } + return oldestElement, nil +} + +// Choose method for priority ejector will return the oldest lowest priority element +func (p *PriorityBidderEjector) Choose(uids map[string]UIDEntry) (string, error) { + nonPriorityUids := getNonPriorityUids(uids, p.PriorityGroups, p.SyncersByBidder) + if err := p.checkSyncerPriority(nonPriorityUids); err != nil { + return "", err + } + + if len(nonPriorityUids) > 0 { + return p.TieEjector.Choose(nonPriorityUids) + } + + lowestPriorityGroup := p.PriorityGroups[len(p.PriorityGroups)-1] + if len(lowestPriorityGroup) == 1 { + uidToDelete := lowestPriorityGroup[0] + p.PriorityGroups = removeElementFromPriorityGroup(p.PriorityGroups, uidToDelete) + return uidToDelete, nil + } + + lowestPriorityUids := getPriorityUids(lowestPriorityGroup, uids, p.SyncersByBidder) + uidToDelete, err := p.TieEjector.Choose(lowestPriorityUids) + if err != nil { + return "", err + } + p.PriorityGroups = removeElementFromPriorityGroup(p.PriorityGroups, uidToDelete) + return uidToDelete, nil +} + +// updatePriorityGroup will remove the selected element from the priority groups, and will remove the entire priority group if it's empty +func removeElementFromPriorityGroup(priorityGroups [][]string, oldestElement string) [][]string { + lowestPriorityGroup := priorityGroups[len(priorityGroups)-1] + if len(lowestPriorityGroup) <= 1 { + return priorityGroups[:len(priorityGroups)-1] + } + + for index, elem := range lowestPriorityGroup { + if elem == oldestElement { + updatedPriorityGroup := append(lowestPriorityGroup[:index], lowestPriorityGroup[index+1:]...) + priorityGroups[len(priorityGroups)-1] = updatedPriorityGroup + return priorityGroups + } + } + return priorityGroups +} + +func getNonPriorityUids(uids map[string]UIDEntry, priorityGroups [][]string, syncersByBidder map[string]Syncer) map[string]UIDEntry { + // If no priority groups, then all keys in uids are non-priority + if len(priorityGroups) == 0 { + return uids + } + + // Create map of keys that are a priority + isPriority := make(map[string]bool) + for _, group := range priorityGroups { + for _, bidder := range group { + if bidderSyncer, ok := syncersByBidder[bidder]; ok { + isPriority[bidderSyncer.Key()] = true + } + } + } + + // Create a map for non-priority uids + nonPriorityUIDs := make(map[string]UIDEntry) + + // Loop over uids and populate the nonPriorityUIDs map with non-priority keys + for key, value := range uids { + if _, found := isPriority[key]; !found { + nonPriorityUIDs[key] = value + } + } + + return nonPriorityUIDs +} + +func getPriorityUids(lowestPriorityGroup []string, uids map[string]UIDEntry, syncersByBidder map[string]Syncer) map[string]UIDEntry { + lowestPriorityUIDs := make(map[string]UIDEntry) + + // Loop over lowestPriorityGroup and populate the lowestPriorityUIDs map + for _, bidder := range lowestPriorityGroup { + if bidderSyncer, ok := syncersByBidder[bidder]; ok { + if uidEntry, exists := uids[bidderSyncer.Key()]; exists { + lowestPriorityUIDs[bidderSyncer.Key()] = uidEntry + } + } + } + return lowestPriorityUIDs +} + +func (p *PriorityBidderEjector) checkSyncerPriority(nonPriorityUids map[string]UIDEntry) error { + if len(nonPriorityUids) == 1 && !p.IsSyncerPriority && len(p.PriorityGroups) > 0 { + return errors.New("syncer key is not a priority, and there are only priority elements left") + } + return nil +} diff --git a/usersync/ejector_test.go b/usersync/ejector_test.go new file mode 100644 index 00000000000..d13e7484b40 --- /dev/null +++ b/usersync/ejector_test.go @@ -0,0 +1,444 @@ +package usersync + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPriorityEjector(t *testing.T) { + testCases := []struct { + name string + givenUids map[string]UIDEntry + givenEjector Ejector + expected string + expectedError error + }{ + { + name: "one-lowest-priority-element", + givenUids: map[string]UIDEntry{ + "highestPrioritySyncer": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + "lowestPriority": { + UID: "456", + Expires: time.Now(), + }, + }, + givenEjector: &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"highestPriorityBidder"}, + {"lowestPriority"}, + }, + SyncersByBidder: map[string]Syncer{ + "highestPriorityBidder": fakeSyncer{ + key: "highestPrioritySyncer", + }, + "lowestPriority": fakeSyncer{ + key: "lowestPriority", + }, + }, + IsSyncerPriority: true, + }, + expected: "lowestPriority", + }, + { + name: "multiple-uids-same-priority", + givenUids: map[string]UIDEntry{ + "newerButSamePriority": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + "olderButSamePriority": { + UID: "456", + Expires: time.Now(), + }, + }, + givenEjector: &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"newerButSamePriority", "olderButSamePriority"}, + }, + SyncersByBidder: map[string]Syncer{ + "newerButSamePriority": fakeSyncer{ + key: "newerButSamePriority", + }, + "olderButSamePriority": fakeSyncer{ + key: "olderButSamePriority", + }, + }, + IsSyncerPriority: true, + TieEjector: &OldestEjector{}, + }, + expected: "olderButSamePriority", + }, + { + name: "non-priority-uids-present", + givenUids: map[string]UIDEntry{ + "higherPriority": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + "lowestPriority": { + UID: "456", + Expires: time.Now(), + }, + "oldestNonPriority": { + UID: "456", + Expires: time.Now(), + }, + "newestNonPriority": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + }, + givenEjector: &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"higherPriority"}, + {"lowestPriority"}, + }, + SyncersByBidder: map[string]Syncer{ + "higherPriority": fakeSyncer{ + key: "higherPriority", + }, + "lowestPriority": fakeSyncer{ + key: "lowestPriority", + }, + "oldestNonPriority": fakeSyncer{ + key: "oldestNonPriority", + }, + "newestNonPriority": fakeSyncer{ + key: "newestNonPriority", + }, + }, + IsSyncerPriority: true, + TieEjector: &OldestEjector{}, + }, + expected: "oldestNonPriority", + }, + { + name: "empty-priority-groups", + givenUids: map[string]UIDEntry{ + "oldestNonPriority": { + UID: "456", + Expires: time.Now(), + }, + "newestNonPriority": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + }, + givenEjector: &PriorityBidderEjector{ + SyncersByBidder: map[string]Syncer{ + "oldestNonPriority": fakeSyncer{ + key: "oldestNonPriority", + }, + "newestNonPriority": fakeSyncer{ + key: "newestNonPriority", + }, + }, + IsSyncerPriority: false, + TieEjector: &OldestEjector{}, + }, + expected: "oldestNonPriority", + }, + { + name: "one-priority-element", + givenUids: map[string]UIDEntry{ + "onlyPriorityElement": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + }, + givenEjector: &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"onlyPriorityElement"}, + }, + SyncersByBidder: map[string]Syncer{ + "onlyPriorityElement": fakeSyncer{ + key: "onlyPriorityElement", + }, + }, + IsSyncerPriority: true, + }, + expected: "onlyPriorityElement", + }, + { + name: "syncer-is-not-priority", + givenUids: map[string]UIDEntry{ + "onlyPriorityElement": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + "syncer": { + UID: "456", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + }, + givenEjector: &PriorityBidderEjector{ + PriorityGroups: [][]string{ + {"onlyPriorityElement"}, + }, + SyncersByBidder: map[string]Syncer{ + "onlyPriorityElement": fakeSyncer{ + key: "onlyPriorityElement", + }, + }, + IsSyncerPriority: false, + }, + expectedError: errors.New("syncer key is not a priority, and there are only priority elements left"), + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + uidToDelete, err := test.givenEjector.Choose(test.givenUids) + if test.expectedError != nil { + assert.Equal(t, test.expectedError, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, uidToDelete) + } + }) + } +} + +func TestOldestEjector(t *testing.T) { + testCases := []struct { + name string + givenUids map[string]UIDEntry + expected string + }{ + { + name: "multiple-elements", + givenUids: map[string]UIDEntry{ + "newestElement": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + "oldestElement": { + UID: "456", + Expires: time.Now(), + }, + }, + expected: "oldestElement", + }, + { + name: "one-element", + givenUids: map[string]UIDEntry{ + "onlyElement": { + UID: "123", + Expires: time.Now().Add((90 * 24 * time.Hour)), + }, + }, + expected: "onlyElement", + }, + { + name: "no-elements", + givenUids: map[string]UIDEntry{}, + expected: "", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + ejector := OldestEjector{} + oldestElement, err := ejector.Choose(test.givenUids) + assert.NoError(t, err) + assert.Equal(t, test.expected, oldestElement) + }) + } +} + +func TestGetNonPriorityUids(t *testing.T) { + syncersByBidder := map[string]Syncer{ + "syncerKey1": fakeSyncer{ + key: "syncerKey1", + }, + "syncerKey2": fakeSyncer{ + key: "syncerKey2", + }, + "syncerKey3": fakeSyncer{ + key: "syncerKey3", + }, + } + + testCases := []struct { + name string + givenUids map[string]UIDEntry + givenPriorityGroups [][]string + expected map[string]UIDEntry + }{ + { + name: "one-priority-group", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + givenPriorityGroups: [][]string{ + {"syncerKey1"}, + }, + expected: map[string]UIDEntry{ + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + }, + { + name: "multiple-priority-groups", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + givenPriorityGroups: [][]string{ + {"syncerKey1"}, + {"syncerKey2"}, + }, + expected: map[string]UIDEntry{ + "syncerKey3": { + UID: "789", + }, + }, + }, + { + name: "no-priority-groups", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + expected: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + uids := getNonPriorityUids(test.givenUids, test.givenPriorityGroups, syncersByBidder) + assert.Equal(t, true, reflect.DeepEqual(test.expected, uids)) + }) + } +} + +func TestGetPriorityUids(t *testing.T) { + syncersByBidder := map[string]Syncer{ + "syncerKey1": fakeSyncer{ + key: "syncerKey1", + }, + "syncerKey2": fakeSyncer{ + key: "syncerKey2", + }, + "syncerKey3": fakeSyncer{ + key: "syncerKey3", + }, + } + + testCases := []struct { + name string + givenUids map[string]UIDEntry + givenLowestPriorityGroup []string + expected map[string]UIDEntry + }{ + { + name: "one-priority-element", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + givenLowestPriorityGroup: []string{"syncerKey1"}, + expected: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + }, + }, + { + name: "multiple-priority-elements", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + givenLowestPriorityGroup: []string{"syncerKey1", "syncerKey2"}, + expected: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + }, + }, + { + name: "no-priority-elements", + givenUids: map[string]UIDEntry{ + "syncerKey1": { + UID: "123", + }, + "syncerKey2": { + UID: "456", + }, + "syncerKey3": { + UID: "789", + }, + }, + givenLowestPriorityGroup: []string{}, + expected: map[string]UIDEntry{}, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + uids := getPriorityUids(test.givenLowestPriorityGroup, test.givenUids, syncersByBidder) + assert.Equal(t, true, reflect.DeepEqual(test.expected, uids)) + }) + } +}