Repository: prebid/prebid-server
Pull request #1422 (2020-07-29 03:48:03)

Validate External Cache Host 
We recently noticed that PrebidJS adds the `http://` scheme to the `ExternalCache` URL `Host`. This pull request presents changes to validate the external cache host. We want to make sure this configuration value does not include a protocol (http, https). If validation fails the server should not start.


Patch:
diff --git a/config/config.go b/config/config.go
index 7fc77855..e3b7d8eb 100755
--- a/config/config.go
+++ b/config/config.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"net/url"
 	"reflect"
@@ -111,6 +112,7 @@ func (cfg *Configuration) validate() configErrors {
 	errs = cfg.CurrencyConverter.validate(errs)
 	errs = validateAdapters(cfg.Adapters, errs)
 	errs = cfg.Debug.validate(errs)
+	errs = cfg.ExtCacheURL.validate(errs)
 	return errs
 }
 
@@ -128,6 +130,40 @@ func (cfg *AuctionTimeouts) validate(errs configErrors) configErrors {
 	return errs
 }
 
+func (data *ExternalCache) validate(errs configErrors) configErrors {
+	if data.Host == "" && data.Path == "" {
+		// Both host and path can be blank. No further validation needed
+		return errs
+	}
+
+	// Either host or path or both not empty, validate.
+	if data.Host == "" && data.Path != "" || data.Host != "" && data.Path == "" {
+		return append(errs, errors.New("External cache Host and Path must both be specified"))
+	}
+	if strings.HasSuffix(data.Host, "/") {
+		return append(errs, errors.New(fmt.Sprintf("External cache Host '%s' must not end with a path separator", data.Host)))
+	}
+	if strings.ContainsAny(data.Host, "://") {
+		return append(errs, errors.New(fmt.Sprintf("External cache Host must not specify a protocol. '%s'", data.Host)))
+	}
+	if !strings.HasPrefix(data.Path, "/") {
+		return append(errs, errors.New(fmt.Sprintf("External cache Path '%s' must begin with a path separator", data.Path)))
+	}
+
+	urlObj, err := url.Parse("https://" + data.Host + data.Path)
+	if err != nil {
+		return append(errs, errors.New(fmt.Sprintf("External cache Path validation error: %s ", err.Error())))
+	}
+	if urlObj.Host != data.Host {
+		return append(errs, errors.New(fmt.Sprintf("External cache Host '%s' is invalid", data.Host)))
+	}
+	if urlObj.Path != data.Path {
+		return append(errs, errors.New("External cache Path is invalid"))
+	}
+
+	return errs
+}
+
 // LimitAuctionTimeout returns the min of requested or cfg.MaxAuctionTimeout.
 // Both values treat "0" as "infinite".
 func (cfg *AuctionTimeouts) LimitAuctionTimeout(requested time.Duration) time.Duration {
@@ -159,6 +195,11 @@ type GDPR struct {
 	TCF1                    TCF1 `mapstructure:"tcf1"`
 	TCF2                    TCF2 `mapstructure:"tcf2"`
 	AMPException            bool `mapstructure:"amp_exception"`
+	// EEACountries (EEA = European Economic Area) are a list of countries where we should assume GDPR applies.
+	// If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only
+	// if the country matches one on this list. If both the GDPR flag and country are not set, we default
+	// to UsersyncIfAmbiguous
+	EEACountries []string `mapstructure:"eea_countries"`
 }
 
 func (cfg *GDPR) validate(errs configErrors) configErrors {
@@ -903,6 +944,10 @@ func SetupViper(v *viper.Viper, filename string) {
 	v.SetDefault("gdpr.tcf2.purpose_one_treatement.enabled", true)
 	v.SetDefault("gdpr.tcf2.purpose_one_treatement.access_allowed", true)
 	v.SetDefault("gdpr.amp_exception", false)
+	v.SetDefault("gdpr.eea_countries", []string{"ALA", "AUT", "BEL", "BGR", "HRV", "CYP", "CZE", "DNK", "EST",
+		"FIN", "FRA", "GUF", "DEU", "GIB", "GRC", "GLP", "GGY", "HUN", "ISL", "IRL", "IMN", "ITA", "JEY", "LVA",
+		"LIE", "LTU", "LUX", "MLT", "MTQ", "MYT", "NLD", "NOR", "POL", "PRT", "REU", "ROU", "BLM", "MAF", "SPM",
+		"SVK", "SVN", "ESP", "SWE", "GBR"})
 	v.SetDefault("ccpa.enforce", false)
 	v.SetDefault("lmt.enforce", true)
 	v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json")
diff --git a/exchange/exchange.go b/exchange/exchange.go
index cf5ec9cc..53f4a7a3 100644
--- a/exchange/exchange.go
+++ b/exchange/exchange.go
@@ -55,6 +55,7 @@ type exchange struct {
 	UsersyncIfAmbiguous bool
 	defaultTTLs         config.DefaultTTLs
 	privacyConfig       config.Privacy
+	eeaCountries        map[string]struct{}
 }
 
 // Container to pass out response ext data from the GetAllBids goroutines back into the main thread
@@ -75,6 +76,10 @@ type bidResponseWrapper struct {
 func NewExchange(client *http.Client, cache prebid_cache_client.Client, cfg *config.Configuration, metricsEngine pbsmetrics.MetricsEngine, infos adapters.BidderInfos, gDPR gdpr.Permissions, currencyConverter *currencies.RateConverter) Exchange {
 	e := new(exchange)
 
+	var s struct{}
+	for _, c := range cfg.GDPR.EEACountries {
+		e.eeaCountries[c] = s
+	}
 	e.adapterMap = newAdapterMap(client, cfg, infos, metricsEngine)
 	e.cache = cache
 	e.cacheTime = time.Duration(cfg.CacheURL.ExpectedTimeMillis) * time.Millisecond
@@ -121,9 +126,27 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque
 		e.me.RecordImps(impLabels)
 	}
 
+	// Make our best guess if GDPR applies
+	usersyncIfAmbiguous := e.UsersyncIfAmbiguous
+	var geo *openrtb.Geo = nil
+	if bidRequest.User != nil && bidRequest.User.Geo != nil {
+		geo = bidRequest.User.Geo
+	} else if bidRequest.Device != nil && bidRequest.Device.Geo != nil {
+		geo = bidRequest.Device.Geo
+	}
+	if geo != nil {
+		// If we have a country set, and it is on the list, we assume GDPR applies if not set on the request.
+		// Otherwise we assume it does not apply as long as it appears "valid" (is 3 characters long).
+		if _, found := e.eeaCountries[strings.ToUpper(geo.Country)]; found {
+			usersyncIfAmbiguous = false
+		} else if len(geo.Country) == 3 {
+			// The country field is formatted properly as a three character country code
+			usersyncIfAmbiguous = true
+		}
+	}
 	// Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder
 	blabels := make(map[openrtb_ext.BidderName]*pbsmetrics.AdapterLabels)
-	cleanRequests, aliases, privacyLabels, errs := cleanOpenRTBRequests(ctx, bidRequest, requestExt, usersyncs, blabels, labels, e.gDPR, e.UsersyncIfAmbiguous, e.privacyConfig)
+	cleanRequests, aliases, privacyLabels, errs := cleanOpenRTBRequests(ctx, bidRequest, requestExt, usersyncs, blabels, labels, e.gDPR, usersyncIfAmbiguous, e.privacyConfig)
 
 	e.me.RecordRequestPrivacy(privacyLabels)
 
diff --git a/gdpr/impl.go b/gdpr/impl.go
index 2deddc7b..2fbd9c5a 100644
--- a/gdpr/impl.go
+++ b/gdpr/impl.go
@@ -228,3 +228,22 @@ func (a AlwaysAllow) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext
 func (a AlwaysAllow) AMPException() bool {
 	return false
 }
+
+// Exporting to allow for easy test setups
+type AlwaysFail struct{}
+
+func (a AlwaysFail) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
+	return false, nil
+}
+
+func (a AlwaysFail) BidderSyncAllowed(ctx context.Context, bidder openrtb_ext.BidderName, consent string) (bool, error) {
+	return false, nil
+}
+
+func (a AlwaysFail) PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, bool, bool, error) {
+	return false, false, false, nil
+}
+
+func (a AlwaysFail) AMPException() bool {
+	return false
+}


Test patch:
diff --git a/config/config_test.go b/config/config_test.go
index 4774d9d6..3da3f721 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -14,6 +14,91 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestExternalCacheURLValidate(t *testing.T) {
+	testCases := []struct {
+		desc      string
+		data      ExternalCache
+		expErrors int
+	}{
+		{
+			desc:      "With http://",
+			data:      ExternalCache{Host: "http://www.google.com", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "Without http://",
+			data:      ExternalCache{Host: "www.google.com", Path: "/path/v1"},
+			expErrors: 0,
+		},
+		{
+			desc:      "No scheme but '//' prefix",
+			data:      ExternalCache{Host: "//www.google.com", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "// appears twice",
+			data:      ExternalCache{Host: "//www.google.com//", Path: "path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "Host has an only // value",
+			data:      ExternalCache{Host: "//", Path: "path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "only scheme host, valid path",
+			data:      ExternalCache{Host: "http://", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "No host, path only",
+			data:      ExternalCache{Host: "", Path: "path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "No host, nor path",
+			data:      ExternalCache{Host: "", Path: ""},
+			expErrors: 0,
+		},
+		{
+			desc:      "Invalid http at the end",
+			data:      ExternalCache{Host: "www.google.com", Path: "http://"},
+			expErrors: 1,
+		},
+		{
+			desc:      "Host has an unknown scheme",
+			data:      ExternalCache{Host: "unknownscheme://host", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "Wrong colon side in scheme",
+			data:      ExternalCache{Host: "http//:www.appnexus.com", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "Missing '/' in scheme",
+			data:      ExternalCache{Host: "http:/www.appnexus.com", Path: "/path/v1"},
+			expErrors: 1,
+		},
+		{
+			desc:      "host with scheme, no path",
+			data:      ExternalCache{Host: "http://www.appnexus.com", Path: ""},
+			expErrors: 1,
+		},
+		{
+			desc:      "scheme, no host nor path",
+			data:      ExternalCache{Host: "http://", Path: ""},
+			expErrors: 1,
+		},
+	}
+	for _, test := range testCases {
+		var errs configErrors
+		errs = test.data.validate(errs)
+
+		assert.Equal(t, test.expErrors, len(errs), "Test case threw unexpected number of errors. Desc: %s errMsg = %v \n", test.desc, errs)
+	}
+}
+
 func TestDefaults(t *testing.T) {
 	v := viper.New()
 	SetupViper(v, "")
@@ -66,7 +151,7 @@ cache:
   query: uuid=%PBS_CACHE_UUID%
 external_cache:
   host: www.externalprebidcache.net
-  path: endpoints/cache
+  path: /endpoints/cache
 http_client:
   max_connections_per_host: 10
   max_idle_connections: 500
@@ -223,7 +308,7 @@ func TestFullConfig(t *testing.T) {
 	cmpStrings(t, "cache.host", cfg.CacheURL.Host, "prebidcache.net")
 	cmpStrings(t, "cache.query", cfg.CacheURL.Query, "uuid=%PBS_CACHE_UUID%")
 	cmpStrings(t, "external_cache.host", cfg.ExtCacheURL.Host, "www.externalprebidcache.net")
-	cmpStrings(t, "external_cache.path", cfg.ExtCacheURL.Path, "endpoints/cache")
+	cmpStrings(t, "external_cache.path", cfg.ExtCacheURL.Path, "/endpoints/cache")
 	cmpInts(t, "http_client.max_connections_per_host", cfg.Client.MaxConnsPerHost, 10)
 	cmpInts(t, "http_client.max_idle_connections", cfg.Client.MaxIdleConns, 500)
 	cmpInts(t, "http_client.max_idle_connections_per_host", cfg.Client.MaxIdleConnsPerHost, 20)
diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go
index 545f04fd..aad448f3 100644
--- a/exchange/exchange_test.go
+++ b/exchange/exchange_test.go
@@ -909,6 +909,9 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) {
 		LMT: config.LMT{
 			Enforce: spec.EnforceLMT,
 		},
+		GDPR: config.GDPR{
+			UsersyncIfAmbiguous: !spec.AssumeGDPRApplies,
+		},
 	}
 
 	ex := newExchangeForTests(t, filename, spec.OutgoingRequests, aliases, privacyConfig)
@@ -1026,15 +1029,22 @@ func newExchangeForTests(t *testing.T, filename string, expectations map[string]
 		}
 	}
 
+	var s struct{}
+	eeac := make(map[string]struct{})
+	for _, c := range []string{"FIN", "FRA", "GUF"} {
+		eeac[c] = s
+	}
+
 	return &exchange{
 		adapterMap:          adapters,
 		me:                  metricsConf.NewMetricsEngine(&config.Configuration{}, openrtb_ext.BidderList()),
 		cache:               &wellBehavedCache{},
 		cacheTime:           0,
-		gDPR:                gdpr.AlwaysAllow{},
+		gDPR:                gdpr.AlwaysFail{},
 		currencyConverter:   currencies.NewRateConverter(&http.Client{}, "", time.Duration(0)),
-		UsersyncIfAmbiguous: false,
+		UsersyncIfAmbiguous: privacyConfig.GDPR.UsersyncIfAmbiguous,
 		privacyConfig:       privacyConfig,
+		eeaCountries:        eeac,
 	}
 }
 
@@ -1882,12 +1892,13 @@ func TestUpdateHbPbCatDur(t *testing.T) {
 }
 
 type exchangeSpec struct {
-	IncomingRequest  exchangeRequest        `json:"incomingRequest"`
-	OutgoingRequests map[string]*bidderSpec `json:"outgoingRequests"`
-	Response         exchangeResponse       `json:"response,omitempty"`
-	EnforceCCPA      bool                   `json:"enforceCcpa"`
-	EnforceLMT       bool                   `json:"enforceLmt"`
-	DebugLog         *DebugLog              `json:"debuglog,omitempty"`
+	IncomingRequest   exchangeRequest        `json:"incomingRequest"`
+	OutgoingRequests  map[string]*bidderSpec `json:"outgoingRequests"`
+	Response          exchangeResponse       `json:"response,omitempty"`
+	EnforceCCPA       bool                   `json:"enforceCcpa"`
+	EnforceLMT        bool                   `json:"enforceLmt"`
+	AssumeGDPRApplies bool                   `json:"assume_gdpr_applies"`
+	DebugLog          *DebugLog              `json:"debuglog,omitempty"`
 }
 
 type exchangeRequest struct {
diff --git a/exchange/exchangetest/gdpr-geo-eu-off-device.json b/exchange/exchangetest/gdpr-geo-eu-off-device.json
new file mode 100644
index 00000000..fc655de8
--- /dev/null
+++ b/exchange/exchangetest/gdpr-geo-eu-off-device.json
@@ -0,0 +1,64 @@
+{
+    "assume_gdpr_applies": false,
+    "incomingRequest": {
+        "ortbRequest": {
+            "id": "some-request-id",
+            "site": {
+                "page": "test.somepage.com"
+            },
+            "imp": [{
+                "id": "my-imp-id",
+                "video": {
+                    "mimes": ["video/mp4"]
+                },
+                "ext": {
+                    "appnexus": {
+                        "placementId": 1
+                    }
+                }
+            }],
+            "user": {
+                "buyeruid": "some-buyer-id"
+            },
+            "device": {
+                "geo": {
+                    "country": "FRA"
+                }
+            }
+}
+    },
+    "outgoingRequests": {
+        "appnexus": {
+            "expectRequest": {
+                "ortbRequest": {
+                    "id": "some-request-id",
+                    "site": {
+                        "page": "test.somepage.com"
+                    },
+                    "imp": [{
+                        "id": "my-imp-id",
+                        "video": {
+                            "mimes": ["video/mp4"]
+                        },
+                        "ext": {
+                            "bidder": {
+                                "placementId": 1
+                            }
+                        }
+                    }],
+                    "user": {
+                    },
+                    "device": {
+                        "geo": {
+                            "country": "FRA"
+                        }
+                    }
+                },
+                "bidAdjustment": 1.0
+            },
+            "mockResponse": {
+                "errors": ["appnexus-error"]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/exchange/exchangetest/gdpr-geo-eu-off.json b/exchange/exchangetest/gdpr-geo-eu-off.json
new file mode 100644
index 00000000..27a030f1
--- /dev/null
+++ b/exchange/exchangetest/gdpr-geo-eu-off.json
@@ -0,0 +1,60 @@
+{
+    "assume_gdpr_applies": false,
+    "incomingRequest": {
+        "ortbRequest": {
+            "id": "some-request-id",
+            "site": {
+                "page": "test.somepage.com"
+            },
+            "imp": [{
+                "id": "my-imp-id",
+                "video": {
+                    "mimes": ["video/mp4"]
+                },
+                "ext": {
+                    "appnexus": {
+                        "placementId": 1
+                    }
+                }
+            }],
+            "user": {
+                "buyeruid": "some-buyer-id",
+                "geo": {
+                    "country": "FRA"
+                }
+            }
+        }
+    },
+    "outgoingRequests": {
+        "appnexus": {
+            "expectRequest": {
+                "ortbRequest": {
+                    "id": "some-request-id",
+                    "site": {
+                        "page": "test.somepage.com"
+                    },
+                    "imp": [{
+                        "id": "my-imp-id",
+                        "video": {
+                            "mimes": ["video/mp4"]
+                        },
+                        "ext": {
+                            "bidder": {
+                                "placementId": 1
+                            }
+                        }
+                    }],
+                    "user": {
+                        "geo": {
+                            "country": "FRA"
+                        }
+                    }
+                },
+                "bidAdjustment": 1.0
+            },
+            "mockResponse": {
+                "errors": ["appnexus-error"]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/exchange/exchangetest/gdpr-geo-eu-on.json b/exchange/exchangetest/gdpr-geo-eu-on.json
new file mode 100644
index 00000000..4ec42fc6
--- /dev/null
+++ b/exchange/exchangetest/gdpr-geo-eu-on.json
@@ -0,0 +1,60 @@
+{
+    "assume_gdpr_applies": true,
+    "incomingRequest": {
+        "ortbRequest": {
+            "id": "some-request-id",
+            "site": {
+                "page": "test.somepage.com"
+            },
+            "imp": [{
+                "id": "my-imp-id",
+                "video": {
+                    "mimes": ["video/mp4"]
+                },
+                "ext": {
+                    "appnexus": {
+                        "placementId": 1
+                    }
+                }
+            }],
+            "user": {
+                "buyeruid": "some-buyer-id",
+                "geo": {
+                    "country": "FRA"
+                }
+            }
+        }
+    },
+    "outgoingRequests": {
+        "appnexus": {
+            "expectRequest": {
+                "ortbRequest": {
+                    "id": "some-request-id",
+                    "site": {
+                        "page": "test.somepage.com"
+                    },
+                    "imp": [{
+                        "id": "my-imp-id",
+                        "video": {
+                            "mimes": ["video/mp4"]
+                        },
+                        "ext": {
+                            "bidder": {
+                                "placementId": 1
+                            }
+                        }
+                    }],
+                    "user": {
+                        "geo": {
+                            "country": "FRA"
+                        }
+                    }
+                },
+                "bidAdjustment": 1.0
+            },
+            "mockResponse": {
+                "errors": ["appnexus-error"]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/exchange/exchangetest/gdpr-geo-usa-off.json b/exchange/exchangetest/gdpr-geo-usa-off.json
new file mode 100644
index 00000000..d56c9318
--- /dev/null
+++ b/exchange/exchangetest/gdpr-geo-usa-off.json
@@ -0,0 +1,61 @@
+{
+    "assume_gdpr_applies": false,
+    "incomingRequest": {
+        "ortbRequest": {
+            "id": "some-request-id",
+            "site": {
+                "page": "test.somepage.com"
+            },
+            "imp": [{
+                "id": "my-imp-id",
+                "video": {
+                    "mimes": ["video/mp4"]
+                },
+                "ext": {
+                    "appnexus": {
+                        "placementId": 1
+                    }
+                }
+            }],
+            "user": {
+                "buyeruid": "some-buyer-id",
+                "geo": {
+                    "country": "USA"
+                }
+            }
+        }
+    },
+    "outgoingRequests": {
+        "appnexus": {
+            "expectRequest": {
+                "ortbRequest": {
+                    "id": "some-request-id",
+                    "site": {
+                        "page": "test.somepage.com"
+                    },
+                    "imp": [{
+                        "id": "my-imp-id",
+                        "video": {
+                            "mimes": ["video/mp4"]
+                        },
+                        "ext": {
+                            "bidder": {
+                                "placementId": 1
+                            }
+                        }
+                    }],
+                    "user": {
+                        "buyeruid": "some-buyer-id",
+                        "geo": {
+                            "country": "USA"
+                        }
+                    }
+                },
+                "bidAdjustment": 1.0
+            },
+            "mockResponse": {
+                "errors": ["appnexus-error"]
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/exchange/exchangetest/gdpr-geo-usa-on.json b/exchange/exchangetest/gdpr-geo-usa-on.json
new file mode 100644
index 00000000..f922be9e
--- /dev/null
+++ b/exchange/exchangetest/gdpr-geo-usa-on.json
@@ -0,0 +1,61 @@
+{
+    "assume_gdpr_applies": true,
+    "incomingRequest": {
+        "ortbRequest": {
+            "id": "some-request-id",
+            "site": {
+                "page": "test.somepage.com"
+            },
+            "imp": [{
+                "id": "my-imp-id",
+                "video": {
+                    "mimes": ["video/mp4"]
+                },
+                "ext": {
+                    "appnexus": {
+                        "placementId": 1
+                    }
+                }
+            }],
+            "user": {
+                "buyeruid": "some-buyer-id",
+                "geo": {
+                    "country": "USA"
+                }
+            }
+        }
+    },
+    "outgoingRequests": {
+        "appnexus": {
+            "expectRequest": {
+                "ortbRequest": {
+                    "id": "some-request-id",
+                    "site": {
+                        "page": "test.somepage.com"
+                    },
+                    "imp": [{
+                        "id": "my-imp-id",
+                        "video": {
+                            "mimes": ["video/mp4"]
+                        },
+                        "ext": {
+                            "bidder": {
+                                "placementId": 1
+                            }
+                        }
+                    }],
+                    "user": {
+                        "buyeruid": "some-buyer-id",
+                        "geo": {
+                            "country": "USA"
+                        }
+                    }
+                },
+                "bidAdjustment": 1.0
+            },
+            "mockResponse": {
+                "errors": ["appnexus-error"]
+            }
+        }
+    }
+}
\ No newline at end of file