Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

UserSync activity #2897

Merged
merged 14 commits into from
Jul 27, 2023
15 changes: 15 additions & 0 deletions endpoints/cookie_sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,13 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, pr
}
}

activityControl, activitiesErr := privacy.NewActivityControl(account.Privacy)
if activitiesErr != nil {
if errortypes.ContainsFatalError([]error{activitiesErr}) {
activityControl = privacy.ActivityControl{}
}
}

syncTypeFilter, err := parseTypeFilter(request.FilterSettings)
if err != nil {
return usersync.Request{}, privacy.Policies{}, err
Expand All @@ -172,6 +179,7 @@ func (c *cookieSyncEndpoint) parseRequest(r *http.Request) (usersync.Request, pr
Privacy: usersyncPrivacy{
gdprPermissions: gdprPerms,
ccpaParsedPolicy: ccpaParsedPolicy,
activityControl: activityControl,
},
SyncTypeFilter: syncTypeFilter,
}
Expand Down Expand Up @@ -501,6 +509,7 @@ type usersyncPrivacyConfig struct {
type usersyncPrivacy struct {
gdprPermissions gdpr.Permissions
ccpaParsedPolicy ccpa.ParsedPolicy
activityControl privacy.ActivityControl
}

func (p usersyncPrivacy) GDPRAllowsHostCookie() bool {
Expand All @@ -517,3 +526,9 @@ func (p usersyncPrivacy) CCPAAllowsBidderSync(bidder string) bool {
enforce := p.ccpaParsedPolicy.CanEnforce() && p.ccpaParsedPolicy.ShouldEnforce(bidder)
return !enforce
}

func (p usersyncPrivacy) ActivityAllowsUserSync(bidder string) bool {
activityResult := p.activityControl.Allow(privacy.ActivitySyncUser,
privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidder})
return activityResult == privacy.ActivityAllow
}
92 changes: 90 additions & 2 deletions endpoints/cookie_sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,7 @@ func TestCookieSyncParseRequest(t *testing.T) {
expectedPrivacy privacy.Policies
expectedRequest usersync.Request
}{

{
description: "Complete Request - includes GPP string with EU TCF V2",
givenBody: strings.NewReader(`{` +
Expand Down Expand Up @@ -973,6 +974,38 @@ func TestCookieSyncParseRequest(t *testing.T) {
expectedError: errCookieSyncAccountBlocked.Error(),
givenAccountRequired: true,
},

{
description: "Account Defaults - Invalid Activities",
givenBody: strings.NewReader(`{` +
`"bidders":["a", "b"],` +
`"account":"ValidAccountInvalidActivities"` +
`}`),
givenGDPRConfig: config.GDPR{Enabled: true, DefaultValue: "0"},
givenCCPAEnabled: true,
givenConfig: config.UserSync{
Cooperative: config.UserSyncCooperative{
EnabledByDefault: false,
PriorityGroups: [][]string{{"a", "b", "c"}},
},
},
expectedPrivacy: privacy.Policies{},
expectedRequest: usersync.Request{
Bidders: []string{"a", "b"},
Cooperative: usersync.Cooperative{
Enabled: false,
PriorityGroups: [][]string{{"a", "b", "c"}},
},
Limit: 0,
Privacy: usersyncPrivacy{
gdprPermissions: &fakePermissions{},
},
SyncTypeFilter: usersync.SyncTypeFilter{
IFrame: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude),
Redirect: usersync.NewUniformBidderFilter(usersync.BidderFilterModeInclude),
},
},
},
}

for _, test := range testCases {
Expand All @@ -997,8 +1030,9 @@ func TestCookieSyncParseRequest(t *testing.T) {
ccpaEnforce: test.givenCCPAEnabled,
},
accountsFetcher: FakeAccountsFetcher{AccountData: map[string]json.RawMessage{
"TestAccount": json.RawMessage(`{"cookie_sync": {"default_limit": 20, "max_limit": 30, "default_coop_sync": true}}`),
"DisabledAccount": json.RawMessage(`{"disabled":true}`),
"TestAccount": json.RawMessage(`{"cookie_sync": {"default_limit": 20, "max_limit": 30, "default_coop_sync": true}}`),
"DisabledAccount": json.RawMessage(`{"disabled":true}`),
"ValidAccountInvalidActivities": json.RawMessage(`{"privacy":{"allowactivities":{"syncUser":{"rules":[{"condition":{"componentName": ["bidderA.bidderB.bidderC"]}}]}}}}`),
}},
}
assert.NoError(t, endpoint.config.MarshalAccountDefaults())
Expand Down Expand Up @@ -1871,6 +1905,41 @@ func TestUsersyncPrivacyCCPAAllowsBidderSync(t *testing.T) {
}
}

func TestCookieSyncActivityControlIntegration(t *testing.T) {
testCases := []struct {
name string
bidderName string
allow bool
expectedResult bool
}{
{
name: "activity_is_allowed",
bidderName: "bidderA",
allow: true,
expectedResult: true,
},
{
name: "activity_is_denied",
bidderName: "bidderA",
allow: false,
expectedResult: false,
},
}

for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
privacyConfig := getDefaultActivityConfig(test.bidderName, test.allow)
activities, err := privacy.NewActivityControl(privacyConfig)
assert.NoError(t, err)
up := usersyncPrivacy{
activityControl: activities,
}
actualResult := up.ActivityAllowsUserSync(test.bidderName)
assert.Equal(t, test.expectedResult, actualResult)
})
}
}

func TestCombineErrors(t *testing.T) {
testCases := []struct {
description string
Expand Down Expand Up @@ -2031,3 +2100,22 @@ func (p *fakePermissions) AuctionActivitiesAllowed(ctx context.Context, bidderCo
AllowBidRequest: true,
}, nil
}

func getDefaultActivityConfig(componentName string, allow bool) *config.AccountPrivacy {
return &config.AccountPrivacy{
AllowActivities: config.AllowActivities{
SyncUser: config.Activity{
Default: ptrutil.ToPtr(true),
Rules: []config.ActivityRule{
{
Allow: allow,
Condition: config.ActivityCondition{
ComponentName: []string{componentName},
ComponentType: []string{"bidder"},
},
},
},
},
},
}
}
31 changes: 23 additions & 8 deletions endpoints/setuid.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/prebid/prebid-server/errortypes"
"github.com/prebid/prebid-server/gdpr"
"github.com/prebid/prebid-server/metrics"
"github.com/prebid/prebid-server/privacy"
gppPrivacy "github.com/prebid/prebid-server/privacy/gpp"
"github.com/prebid/prebid-server/stored_requests"
"github.com/prebid/prebid-server/usersync"
Expand Down Expand Up @@ -65,7 +66,7 @@ func NewSetUIDEndpoint(cfg *config.Configuration, syncersByBidder map[string]use

query := r.URL.Query()

syncer, err := getSyncer(query, syncersByKey)
syncer, bidderName, err := getSyncer(query, syncersByKey)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
Expand Down Expand Up @@ -110,6 +111,20 @@ func NewSetUIDEndpoint(cfg *config.Configuration, syncersByBidder map[string]use
return
}

activities, activitiesErr := privacy.NewActivityControl(account.Privacy)
if activitiesErr != nil {
if errortypes.ContainsFatalError([]error{activitiesErr}) {
activities = privacy.ActivityControl{}
}
}

userSyncActivityAllowed := activities.Allow(privacy.ActivitySyncUser,
privacy.ScopedName{Scope: privacy.ScopeTypeBidder, Name: bidderName})
if userSyncActivityAllowed == privacy.ActivityDeny {
w.WriteHeader(http.StatusUnavailableForLegalReasons)
return
Comment on lines +117 to +118
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably record a metric and update the analytics object so here as well.
For GDPR, the metric we use when the error is http.StatusUnavailableForLegalReasons is metrics.SetUidGDPRHostCookieBlocked. Maybe we need to add a more generic privacy related blocking metric like metrics.SetUidPrivacyHostCookieBlocked?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We discussed this offline and decided to add metrics in a separate PR.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using a generic metric is a good idea, especially as the number of privacy policies is expected to increase. IMHO the existign GDPR and CCPA metrics could use the new generalized privacy blocking metric.

}

gdprRequestInfo, err := extractGDPRInfo(query)
if err != nil {
// Only exit if non-warning
Expand Down Expand Up @@ -313,19 +328,19 @@ func parseConsentFromGppStr(gppQueryValue string) (string, error) {
return gdprConsent, nil
}

func getSyncer(query url.Values, syncersByKey map[string]usersync.Syncer) (usersync.Syncer, error) {
key := query.Get("bidder")
func getSyncer(query url.Values, syncersByKey map[string]usersync.Syncer) (usersync.Syncer, string, error) {
bidderName := query.Get("bidder")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't believe this change is valid. The "bidder" parameter is actually the syncer key. There is often a 1:1 relationship between them, but it's possible to have a 1:many.

Let's discuss at our next meeting how to address this, since we need the proper bidder name now for activity rule matching. Since we only ever expect the setuid endpoint to be called as a redirect from a cookie sync, we should have the freedom to make this change.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch Scott, yeah this function gets passed a query from a cookie sync redirect, where "bidder" would actually grab the syncer key. Maybe that syncersByBidder map that gets passed into NewSetUIDEndpoint() can be used to get the proper bidder name?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good though. The problem is there is a 1:many relationship from the syncer key to the bidder name, so we can't be sure which bidder it is. We'll likely be fine if at least one of the linked bidders is allowed, but that's complicated to check. I think the easier approach would be to actually set bidder to the bidder name. SetUID would still need the syncersByBidder map to get the syncer key.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow! Let's discuss it in a team meeting. I'll follow your instructions here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I saw that you closed your bidderNameBySyncer PR. Will your changes be reflected in this PR or will you open a new one for this?


if key == "" {
return nil, errors.New(`"bidder" query param is required`)
if bidderName == "" {
return nil, "", errors.New(`"bidder" query param is required`)
}

syncer, syncerExists := syncersByKey[key]
syncer, syncerExists := syncersByKey[bidderName]
if !syncerExists {
return nil, errors.New("The bidder name provided is not supported by Prebid Server")
return nil, "", errors.New("The bidder name provided is not supported by Prebid Server")
}

return syncer, nil
return syncer, bidderName, nil
}

// getResponseFormat reads the format query parameter or falls back to the syncer's default.
Expand Down
33 changes: 33 additions & 0 deletions endpoints/setuid_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,35 @@ func TestSetUIDEndpoint(t *testing.T) {
expectedBody: "account is disabled, please reach out to the prebid server host",
description: "Set uid for valid bidder with valid disabled account provided",
},
{
uri: "/setuid?bidder=pubmatic&uid=123&account=valid_acct_with_valid_activities_usersync_enabled",
syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"},
existingSyncs: nil,
gdprAllowsHostCookies: true,
expectedSyncs: map[string]string{"pubmatic": "123"},
expectedStatusCode: http.StatusOK,
expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"},
description: "Set uid for valid bidder with valid account provided with user sync allowed activity",
},
{
uri: "/setuid?bidder=pubmatic&uid=123&account=valid_acct_with_valid_activities_usersync_disabled",
syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"},
existingSyncs: nil,
gdprAllowsHostCookies: true,
expectedSyncs: nil,
expectedStatusCode: http.StatusUnavailableForLegalReasons,
description: "Set uid for valid bidder with valid account provided with user sync disallowed activity",
},
{
uri: "/setuid?bidder=pubmatic&uid=123&account=valid_acct_with_invalid_activities",
syncersBidderNameToKey: map[string]string{"pubmatic": "pubmatic"},
existingSyncs: nil,
gdprAllowsHostCookies: true,
expectedSyncs: map[string]string{"pubmatic": "123"},
expectedStatusCode: http.StatusOK,
expectedHeaders: map[string]string{"Content-Type": "text/html", "Content-Length": "0"},
description: "Set uid for valid bidder with valid account provided with invalid user sync activity",
},
}

analytics := analyticsConf.NewPBSAnalytics(&config.Analytics{})
Expand Down Expand Up @@ -1354,6 +1383,10 @@ func doRequest(req *http.Request, analytics analytics.PBSAnalyticsModule, metric
"disabled_acct": json.RawMessage(`{"disabled":true}`),
"malformed_acct": json.RawMessage(`{"disabled":"malformed"}`),
"invalid_json_acct": json.RawMessage(`{"}`),

"valid_acct_with_valid_activities_usersync_enabled": json.RawMessage(`{"privacy":{"allowactivities":{"syncUser":{"default": true}}}}`),
"valid_acct_with_valid_activities_usersync_disabled": json.RawMessage(`{"privacy":{"allowactivities":{"syncUser":{"default": false}}}}`),
"valid_acct_with_invalid_activities": json.RawMessage(`{"privacy":{"allowactivities":{"syncUser":{"rules":[{"condition":{"componentName": ["bidderA.bidderB.bidderC"]}}]}}}}`),
}}

endpoint := NewSetUIDEndpoint(&cfg, syncersByBidder, gdprPermsBuilder, tcf2ConfigBuilder, analytics, fakeAccountsFetcher, metrics)
Expand Down
6 changes: 1 addition & 5 deletions privacy/enforcer.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package privacy
import (
"fmt"
"github.com/prebid/prebid-server/config"
"github.com/prebid/prebid-server/errortypes"
"strings"
)

Expand Down Expand Up @@ -32,10 +31,7 @@ func NewActivityControl(privacyConf *config.AccountPrivacy) (ActivityControl, er
var err error

if privacyConf == nil {
return ac, err
} else {
//temporarily disable Activities if they are specified at the account level
return ac, &errortypes.Warning{Message: "account.Privacy has no effect as the feature is under development."}
return ac, nil
}

plans := make(map[Activity]ActivityPlan)
Expand Down
2 changes: 1 addition & 1 deletion privacy/enforcer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"testing"
)

func TemporarilyDisabledTestNewActivityControl(t *testing.T) {
func TestNewActivityControl(t *testing.T) {

testCases := []struct {
name string
Expand Down
9 changes: 9 additions & 0 deletions usersync/chooser.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,17 @@ const (

// StatusDuplicate specifies the bidder is a duplicate or shared a syncer key with another bidder choice.
StatusDuplicate

// StatusBlockedByPrivacy specifies a bidder sync url is not allowed by privacy activities
StatusBlockedByPrivacy
)

// Privacy determines which privacy policies will be enforced for a user sync request.
type Privacy interface {
GDPRAllowsHostCookie() bool
GDPRAllowsBidderSync(bidder string) bool
CCPAAllowsBidderSync(bidder string) bool
ActivityAllowsUserSync(bidder string) bool
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we rename to ActivityAllowsBidderSync to match GDPRAllowsBidderSync and CCPAAllowsBidderSync?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gus, activity name we execute here is syncUser :

case ActivitySyncUser:

and this is how it is in configs:
SyncUser Activity `mapstructure:"syncUser" json:"syncUser"`

I'm open to rename it if you think ActivityAllowsBidderSync is better than ActivityAllowsUserSync.

}

// standardChooser implements the user syncer algorithm per official Prebid specification.
Expand Down Expand Up @@ -151,6 +155,11 @@ func (c standardChooser) evaluate(bidder string, syncersSeen map[string]struct{}
return nil, BidderEvaluation{Status: StatusAlreadySynced, Bidder: bidder, SyncerKey: syncer.Key()}
}

userSyncActivityAllowed := privacy.ActivityAllowsUserSync(bidder)
if !userSyncActivityAllowed {
return nil, BidderEvaluation{Status: StatusBlockedByPrivacy, Bidder: bidder, SyncerKey: syncer.Key()}
}

if !privacy.GDPRAllowsBidderSync(bidder) {
return nil, BidderEvaluation{Status: StatusBlockedByGDPR, Bidder: bidder, SyncerKey: syncer.Key()}
}
Expand Down
Loading
Loading