diff --git a/config/config.go b/config/config.go index 203dcd57c38..562b70af60d 100644 --- a/config/config.go +++ b/config/config.go @@ -204,6 +204,16 @@ type Cache struct { // this should be replaced by code which tracks the response time of recent cache calls and // adjusts the time dynamically. ExpectedTimeMillis int `mapstructure:"expected_millis"` + + DefaultTTLs DefaultTTLs `mapstructure:"default_ttl_seconds"` +} + +// Default TTLs to use to cache bids for different types of imps. +type DefaultTTLs struct { + Banner int `mapstructure:"banner"` + Video int `mapstructure:"video"` + Native int `mapstructure:"native"` + Audio int `mapstructure:"audio"` } type Cookie struct { @@ -261,6 +271,10 @@ func SetupViper(v *viper.Viper, filename string) { v.SetDefault("cache.host", "") v.SetDefault("cache.query", "") v.SetDefault("cache.expected_millis", 10) + v.SetDefault("cache.default_ttl_seconds.banner", 0) + v.SetDefault("cache.default_ttl_seconds.video", 0) + v.SetDefault("cache.default_ttl_seconds.native", 0) + v.SetDefault("cache.default_ttl_seconds.audio", 0) v.SetDefault("recaptcha_secret", "") v.SetDefault("host_cookie.domain", "") v.SetDefault("host_cookie.family", "") diff --git a/exchange/auction.go b/exchange/auction.go index 9ad97de327b..7364e765d5b 100644 --- a/exchange/auction.go +++ b/exchange/auction.go @@ -6,6 +6,7 @@ import ( "github.com/golang/glog" "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" ) @@ -55,7 +56,7 @@ func (a *auction) setRoundedPrices(priceGranularity openrtb_ext.PriceGranularity a.roundedPrices = roundedPrices } -func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, bids bool, vast bool, bidRequest *openrtb.BidRequest, ttlBuffer int64) []error { +func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, bids bool, vast bool, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs) []error { if !bids && !vast { return nil } @@ -78,7 +79,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, toCache = append(toCache, prebid_cache_client.Cacheable{ Type: prebid_cache_client.TypeJSON, Data: jsonBytes, - TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, ttlBuffer), + TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, defTTL(topBidPerBidder.bidType, defaultTTLs), ttlBuffer), }) bidIndices[len(toCache)-1] = topBidPerBidder.bid } @@ -89,7 +90,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, toCache = append(toCache, prebid_cache_client.Cacheable{ Type: prebid_cache_client.TypeXML, Data: jsonBytes, - TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, ttlBuffer), + TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, defTTL(topBidPerBidder.bidType, defaultTTLs), ttlBuffer), }) vastIndices[len(toCache)-1] = topBidPerBidder.bid } @@ -145,7 +146,12 @@ func maybeMake(shouldMake bool, capacity int) []prebid_cache_client.Cacheable { return nil } -func cacheTTL(impTTL int64, bidTTL int64, buffer int64) (ttl int64) { +func cacheTTL(impTTL int64, bidTTL int64, defTTL int64, buffer int64) (ttl int64) { + if impTTL <= 0 && bidTTL <= 0 { + // Only use default if there is no imp nor bid TTL provided. We don't want the default + // to cut short a requested longer TTL. + return addBuffer(defTTL, buffer) + } if impTTL <= 0 { // Use <= to handle the case of someone sending a negative ttl. We treat it as zero return addBuffer(bidTTL, buffer) @@ -166,6 +172,20 @@ func addBuffer(base int64, buffer int64) int64 { return base + buffer } +func defTTL(bidType openrtb_ext.BidType, defaultTTLs *config.DefaultTTLs) (ttl int64) { + switch bidType { + case openrtb_ext.BidTypeBanner: + return int64(defaultTTLs.Banner) + case openrtb_ext.BidTypeVideo: + return int64(defaultTTLs.Video) + case openrtb_ext.BidTypeNative: + return int64(defaultTTLs.Native) + case openrtb_ext.BidTypeAudio: + return int64(defaultTTLs.Audio) + } + return 0 +} + type auction struct { // winningBids is a map from imp.id to the highest overall CPM bid in that imp. winningBids map[string]*pbsOrtbBid diff --git a/exchange/auction_test.go b/exchange/auction_test.go index 2b57fe20149..f8daa2ae1e3 100644 --- a/exchange/auction_test.go +++ b/exchange/auction_test.go @@ -4,8 +4,11 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" + "strconv" "testing" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/prebid_cache_client" @@ -37,102 +40,66 @@ func TestMakeVASTNurl(t *testing.T) { assert.Equal(t, expect, vast) } -func TestDoCache(t *testing.T) { - bidRequest := &openrtb.BidRequest{ - Imp: []openrtb.Imp{ - { - ID: "oneImp", - Exp: 300, - }, - { - ID: "twoImp", - }, - }, - } - bid := make([]pbsOrtbBid, 5) - winningBidsByBidder := make(map[string]map[openrtb_ext.BidderName]*pbsOrtbBid) - roundedPrices := make(map[*pbsOrtbBid]string) - winningBidsByBidder["oneImp"] = make(map[openrtb_ext.BidderName]*pbsOrtbBid) - bid[0] = pbsOrtbBid{ - bid: &openrtb.Bid{ - Price: 7.64, - Exp: 600, - }, - } - winningBidsByBidder["oneImp"][openrtb_ext.BidderAppnexus] = &bid[0] - roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderAppnexus]] = "7.64" - bid[1] = pbsOrtbBid{ - bid: &openrtb.Bid{ - Price: 5.64, - Exp: 200, - }, - } - winningBidsByBidder["oneImp"][openrtb_ext.BidderPubmatic] = &bid[1] - roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderPubmatic]] = "5.64" - bid[2] = pbsOrtbBid{ - bid: &openrtb.Bid{ - Price: 2.3, - }, +// TestCacheJSON executes tests for all the *.json files in cachetest. +func TestCacheJSON(t *testing.T) { + if specFiles, err := ioutil.ReadDir("./cachetest"); err == nil { + for _, specFile := range specFiles { + fileName := "./cachetest/" + specFile.Name() + fileDisplayName := "exchange/cachetest/" + specFile.Name() + specData, err := loadCacheSpec(fileName) + if err != nil { + t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err) + } + + runCacheSpec(t, fileDisplayName, specData) + } } - winningBidsByBidder["oneImp"][openrtb_ext.BidderOpenx] = &bid[2] - roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderOpenx]] = "2.3" - winningBidsByBidder["twoImp"] = make(map[openrtb_ext.BidderName]*pbsOrtbBid) - bid[3] = pbsOrtbBid{ - bid: &openrtb.Bid{ - Price: 1.64, - }, +} + +// LoadCacheSpec reads and parses a file as a test case. If something goes wrong, it returns an error. +func loadCacheSpec(filename string) (*cacheSpec, error) { + specData, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("Failed to read file %s: %v", filename, err) } - winningBidsByBidder["twoImp"][openrtb_ext.BidderAppnexus] = &bid[3] - roundedPrices[winningBidsByBidder["twoImp"][openrtb_ext.BidderAppnexus]] = "1.64" - bid[4] = pbsOrtbBid{ - bid: &openrtb.Bid{ - Price: 7.64, - Exp: 900, - }, + + var spec cacheSpec + if err := json.Unmarshal(specData, &spec); err != nil { + return nil, fmt.Errorf("Failed to unmarshal JSON from file: %v", err) } - winningBidsByBidder["twoImp"][openrtb_ext.BidderRubicon] = &bid[4] - roundedPrices[winningBidsByBidder["twoImp"][openrtb_ext.BidderRubicon]] = "7.64" - testAuction := &auction{ - winningBidsByBidder: winningBidsByBidder, + + return &spec, nil +} + +func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) { + // bid := make([]pbsOrtbBid, 5) + var bid *pbsOrtbBid + winningBidsByBidder := make(map[string]map[openrtb_ext.BidderName]*pbsOrtbBid) + roundedPrices := make(map[*pbsOrtbBid]string) + for i, pbsBid := range specData.PbsBids { + if _, ok := winningBidsByBidder[pbsBid.Bid.ID]; !ok { + winningBidsByBidder[pbsBid.Bid.ID] = make(map[openrtb_ext.BidderName]*pbsOrtbBid) + } + bid = &pbsOrtbBid{ + bid: pbsBid.Bid, + bidType: pbsBid.BidType, + } + winningBidsByBidder[pbsBid.Bid.ID][pbsBid.Bidder] = bid + roundedPrices[bid] = strconv.FormatFloat(bid.bid.Price, 'f', 2, 64) + // Marshal the bid for the expected cacheables + cjson, _ := json.Marshal(bid.bid) + specData.ExpectedCacheables[i].Data = cjson } ctx := context.Background() cache := &mockCache{} - _ = testAuction.doCache(ctx, cache, true, false, bidRequest, 60) - json0, _ := json.Marshal(bid[0].bid) - json1, _ := json.Marshal(bid[1].bid) - json2, _ := json.Marshal(bid[2].bid) - json3, _ := json.Marshal(bid[3].bid) - json4, _ := json.Marshal(bid[4].bid) - cacheables := make([]prebid_cache_client.Cacheable, 5) - cacheables[0] = prebid_cache_client.Cacheable{ - Type: prebid_cache_client.TypeJSON, - TTLSeconds: 360, - Data: json0, - } - cacheables[1] = prebid_cache_client.Cacheable{ - Type: prebid_cache_client.TypeJSON, - TTLSeconds: 260, - Data: json1, - } - cacheables[2] = prebid_cache_client.Cacheable{ - Type: prebid_cache_client.TypeJSON, - TTLSeconds: 360, - Data: json2, - } - cacheables[3] = prebid_cache_client.Cacheable{ - Type: prebid_cache_client.TypeJSON, - TTLSeconds: 0, - Data: json3, - } - cacheables[4] = prebid_cache_client.Cacheable{ - Type: prebid_cache_client.TypeJSON, - TTLSeconds: 960, - Data: json4, + testAuction := &auction{ + winningBidsByBidder: winningBidsByBidder, } + _ = testAuction.doCache(ctx, cache, true, false, &specData.BidRequest, 60, &specData.DefaultTTLs) found := 0 - for _, cExpected := range cacheables { + for _, cExpected := range specData.ExpectedCacheables { for _, cFound := range cache.items { eq := jsonpatch.Equal(cExpected.Data, cFound.Data) if cExpected.TTLSeconds == cFound.TTLSeconds && eq { @@ -141,14 +108,27 @@ func TestDoCache(t *testing.T) { } } - if found != 5 { - fmt.Printf("Expected:\n%v\n\n", cacheables) + if found != len(specData.ExpectedCacheables) { + fmt.Printf("Expected:\n%v\n\n", specData.ExpectedCacheables) fmt.Printf("Found:\n%v\n\n", cache.items) - t.Errorf("All expected cacheables not found. Expected 5, found %d.", found) + t.Errorf("All expected cacheables not found. Expected %d, found %d.", len(specData.ExpectedCacheables), found) } } +type cacheSpec struct { + BidRequest openrtb.BidRequest `json:"bidRequest"` + PbsBids []pbsBid `json:"pbsBids"` + ExpectedCacheables []prebid_cache_client.Cacheable `json:"expectedCacheables"` + DefaultTTLs config.DefaultTTLs `json:"defaultTTLs"` +} + +type pbsBid struct { + Bid *openrtb.Bid `json:"bid"` + BidType openrtb_ext.BidType `json:"bidType"` + Bidder openrtb_ext.BidderName `json:"bidder"` +} + type mockCache struct { items []prebid_cache_client.Cacheable } diff --git a/exchange/cachetest/defaultbanner.json b/exchange/cachetest/defaultbanner.json new file mode 100644 index 00000000000..202bb3b3771 --- /dev/null +++ b/exchange/cachetest/defaultbanner.json @@ -0,0 +1,41 @@ +{ + "bidRequest": { + "imp": [ + { + "id": "oneImp" + } + ] + }, + "pbsBids": [{ + "bid":{ + "id": "oneImp", + "price": 7.64, + "exp": 600 + }, + "bidType": "banner", + "bidder": "appnexus" + }, { + "bid": { + "id": "oneImp", + "price": 5.64 + }, + "bidType": "banner", + "bidder": "pubmatic" + }], + "expectedCacheables": [ + { + "Type": "json", + "TTLSeconds": 660 + }, { + "Type": "json", + "TTLSeconds": 360 + } + ], + "defaultTTLs": { + "banner": 300, + "video": 3600, + "audio": 1800, + "native": 300 + } +} + diff --git a/exchange/cachetest/defaultvideo.json b/exchange/cachetest/defaultvideo.json new file mode 100644 index 00000000000..1b4c64934d2 --- /dev/null +++ b/exchange/cachetest/defaultvideo.json @@ -0,0 +1,42 @@ +{ + "bidRequest": { + "imp": [ + { + "id": "oneImp", + "exp": 600 + }, { + "id": "twoImp" + } + ] + }, + "pbsBids": [{ + "bid":{ + "id": "oneImp", + "price": 7.64 + }, + "bidType": "video", + "bidder": "appnexus" + }, { + "bid": { + "id": "twoImp", + "price": 5.64 + }, + "bidType": "video", + "bidder": "pubmatic" + }], + "expectedCacheables": [ + { + "Type": "json", + "TTLSeconds": 660 + }, { + "Type": "json", + "TTLSeconds": 3660 + } + ], + "defaultTTLs": { + "banner": 300, + "video": 3600, + "audio": 1800, + "native": 300 + } +} \ No newline at end of file diff --git a/exchange/cachetest/multibid.json b/exchange/cachetest/multibid.json new file mode 100644 index 00000000000..ecfecba238f --- /dev/null +++ b/exchange/cachetest/multibid.json @@ -0,0 +1,71 @@ +{ + "bidRequest": { + "imp": [ + { + "id": "oneImp", + "exp": 300 + }, + { + "id": "twoImp" + } + ] + }, + "pbsBids": [{ + "bid":{ + "id": "oneImp", + "price": 7.64, + "exp": 600 + }, + "bidType": "banner", + "bidder": "appnexus" + }, { + "bid": { + "id": "oneImp", + "price": 5.64, + "exp": 200 + }, + "bidType": "banner", + "bidder": "pubmatic" + }, { + "bid": { + "id": "oneImp", + "price": 2.3 + }, + "bidType": "banner", + "bidder": "openx" + }, { + "bid": { + "id": "twoImp", + "price": 1.64 + }, + "bidType": "banner", + "bidder": "appnexus" + }, { + "bid": { + "id": "twoImp", + "price": 7.64, + "exp": 900 + }, + "bidType": "banner", + "bidder": "rubicon" + } + ], + "expectedCacheables": [ + { + "Type": "json", + "TTLSeconds": 360 + }, { + "Type": "json", + "TTLSeconds": 260 + }, { + "Type": "json", + "TTLSeconds": 360 + }, { + "Type": "json", + "TTLSeconds": 0 + }, { + "Type": "json", + "TTLSeconds": 960 + } + ] +} \ No newline at end of file diff --git a/exchange/exchange.go b/exchange/exchange.go index b2a8f1966bb..48f7c105a10 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -42,6 +42,7 @@ type exchange struct { cacheTime time.Duration gDPR gdpr.Permissions UsersyncIfAmbiguous bool + defaultTTLs config.DefaultTTLs } // Container to pass out response ext data from the GetAllBids goroutines back into the main thread @@ -65,6 +66,7 @@ func NewExchange(client *http.Client, cache prebid_cache_client.Client, cfg *con e.me = metricsEngine e.gDPR = gDPR e.UsersyncIfAmbiguous = cfg.GDPR.UsersyncIfAmbiguous + e.defaultTTLs = cfg.CacheURL.DefaultTTLs return e } @@ -133,7 +135,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque auc := newAuction(adapterBids, len(bidRequest.Imp)) if targData != nil { auc.setRoundedPrices(targData.priceGranularity) - cacheErrs := auc.doCache(ctx, e.cache, targData.includeCacheBids, targData.includeCacheVast, bidRequest, 60) + cacheErrs := auc.doCache(ctx, e.cache, targData.includeCacheBids, targData.includeCacheVast, bidRequest, 60, &e.defaultTTLs) if len(cacheErrs) > 0 { errs = append(errs, cacheErrs...) }