Default ttl per media type (#697)

* Adds support for default cache TTLs per Imp media types

* Fixes bugs in tests

* Fixes issues identified in PR review:

* Refactors existing test into a JSON driven format, and adds new tests

config/config.go

@@ -204,6 +204,16 @@ type Cache struct {
 	// this should be replaced by code which tracks the response time of recent cache calls and
 	// adjusts the time dynamically.
 	ExpectedTimeMillis int `mapstructure:"expected_millis"`
+
+	DefaultTTLs DefaultTTLs `mapstructure:"default_ttl_seconds"`
+}
+
+// Default TTLs to use to cache bids for different types of imps.
+type DefaultTTLs struct {
+	Banner int `mapstructure:"banner"`
+	Video  int `mapstructure:"video"`
+	Native int `mapstructure:"native"`
+	Audio  int `mapstructure:"audio"`
 }
 
 type Cookie struct {
@@ -261,6 +271,10 @@ func SetupViper(v *viper.Viper, filename string) {
 	v.SetDefault("cache.host", "")
 	v.SetDefault("cache.query", "")
 	v.SetDefault("cache.expected_millis", 10)
+	v.SetDefault("cache.default_ttl_seconds.banner", 0)
+	v.SetDefault("cache.default_ttl_seconds.video", 0)
+	v.SetDefault("cache.default_ttl_seconds.native", 0)
+	v.SetDefault("cache.default_ttl_seconds.audio", 0)
 	v.SetDefault("recaptcha_secret", "")
 	v.SetDefault("host_cookie.domain", "")
 	v.SetDefault("host_cookie.family", "")

exchange/auction.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/mxmCherry/openrtb"
+	"github.com/prebid/prebid-server/config"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/prebid_cache_client"
 )
@@ -55,7 +56,7 @@ func (a *auction) setRoundedPrices(priceGranularity openrtb_ext.PriceGranularity
 	a.roundedPrices = roundedPrices
 }
 
-func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, bids bool, vast bool, bidRequest *openrtb.BidRequest, ttlBuffer int64) []error {
+func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client, bids bool, vast bool, bidRequest *openrtb.BidRequest, ttlBuffer int64, defaultTTLs *config.DefaultTTLs) []error {
 	if !bids && !vast {
 		return nil
 	}
@@ -78,7 +79,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
 					toCache = append(toCache, prebid_cache_client.Cacheable{
 						Type:       prebid_cache_client.TypeJSON,
 						Data:       jsonBytes,
-						TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, ttlBuffer),
+						TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, defTTL(topBidPerBidder.bidType, defaultTTLs), ttlBuffer),
 					})
 					bidIndices[len(toCache)-1] = topBidPerBidder.bid
 				}
@@ -89,7 +90,7 @@ func (a *auction) doCache(ctx context.Context, cache prebid_cache_client.Client,
 					toCache = append(toCache, prebid_cache_client.Cacheable{
 						Type:       prebid_cache_client.TypeXML,
 						Data:       jsonBytes,
-						TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, ttlBuffer),
+						TTLSeconds: cacheTTL(expByImp[impID], topBidPerBidder.bid.Exp, defTTL(topBidPerBidder.bidType, defaultTTLs), ttlBuffer),
 					})
 					vastIndices[len(toCache)-1] = topBidPerBidder.bid
 				}
@@ -145,7 +146,12 @@ func maybeMake(shouldMake bool, capacity int) []prebid_cache_client.Cacheable {
 	return nil
 }
 
-func cacheTTL(impTTL int64, bidTTL int64, buffer int64) (ttl int64) {
+func cacheTTL(impTTL int64, bidTTL int64, defTTL int64, buffer int64) (ttl int64) {
+	if impTTL <= 0 && bidTTL <= 0 {
+		// Only use default if there is no imp nor bid TTL provided. We don't want the default
+		// to cut short a requested longer TTL.
+		return addBuffer(defTTL, buffer)
+	}
 	if impTTL <= 0 {
 		// Use <= to handle the case of someone sending a negative ttl. We treat it as zero
 		return addBuffer(bidTTL, buffer)
@@ -166,6 +172,20 @@ func addBuffer(base int64, buffer int64) int64 {
 	return base + buffer
 }
 
+func defTTL(bidType openrtb_ext.BidType, defaultTTLs *config.DefaultTTLs) (ttl int64) {
+	switch bidType {
+	case openrtb_ext.BidTypeBanner:
+		return int64(defaultTTLs.Banner)
+	case openrtb_ext.BidTypeVideo:
+		return int64(defaultTTLs.Video)
+	case openrtb_ext.BidTypeNative:
+		return int64(defaultTTLs.Native)
+	case openrtb_ext.BidTypeAudio:
+		return int64(defaultTTLs.Audio)
+	}
+	return 0
+}
+
 type auction struct {
 	// winningBids is a map from imp.id to the highest overall CPM bid in that imp.
 	winningBids map[string]*pbsOrtbBid

exchange/auction_test.go

@@ -4,8 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"strconv"
 	"testing"
 
+	"github.com/prebid/prebid-server/config"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/prebid_cache_client"
 
@@ -37,102 +40,66 @@ func TestMakeVASTNurl(t *testing.T) {
 	assert.Equal(t, expect, vast)
 }
 
-func TestDoCache(t *testing.T) {
-	bidRequest := &openrtb.BidRequest{
-		Imp: []openrtb.Imp{
-			{
-				ID:  "oneImp",
-				Exp: 300,
-			},
-			{
-				ID: "twoImp",
-			},
-		},
-	}
-	bid := make([]pbsOrtbBid, 5)
-	winningBidsByBidder := make(map[string]map[openrtb_ext.BidderName]*pbsOrtbBid)
-	roundedPrices := make(map[*pbsOrtbBid]string)
-	winningBidsByBidder["oneImp"] = make(map[openrtb_ext.BidderName]*pbsOrtbBid)
-	bid[0] = pbsOrtbBid{
-		bid: &openrtb.Bid{
-			Price: 7.64,
-			Exp:   600,
-		},
-	}
-	winningBidsByBidder["oneImp"][openrtb_ext.BidderAppnexus] = &bid[0]
-	roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderAppnexus]] = "7.64"
-	bid[1] = pbsOrtbBid{
-		bid: &openrtb.Bid{
-			Price: 5.64,
-			Exp:   200,
-		},
-	}
-	winningBidsByBidder["oneImp"][openrtb_ext.BidderPubmatic] = &bid[1]
-	roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderPubmatic]] = "5.64"
-	bid[2] = pbsOrtbBid{
-		bid: &openrtb.Bid{
-			Price: 2.3,
-		},
+// TestCacheJSON executes tests for all the *.json files in cachetest.
+func TestCacheJSON(t *testing.T) {
+	if specFiles, err := ioutil.ReadDir("./cachetest"); err == nil {
+		for _, specFile := range specFiles {
+			fileName := "./cachetest/" + specFile.Name()
+			fileDisplayName := "exchange/cachetest/" + specFile.Name()
+			specData, err := loadCacheSpec(fileName)
+			if err != nil {
+				t.Fatalf("Failed to load contents of file %s: %v", fileDisplayName, err)
+			}
+
+			runCacheSpec(t, fileDisplayName, specData)
+		}
 	}
-	winningBidsByBidder["oneImp"][openrtb_ext.BidderOpenx] = &bid[2]
-	roundedPrices[winningBidsByBidder["oneImp"][openrtb_ext.BidderOpenx]] = "2.3"
-	winningBidsByBidder["twoImp"] = make(map[openrtb_ext.BidderName]*pbsOrtbBid)
-	bid[3] = pbsOrtbBid{
-		bid: &openrtb.Bid{
-			Price: 1.64,
-		},
+}
+
+// LoadCacheSpec reads and parses a file as a test case. If something goes wrong, it returns an error.
+func loadCacheSpec(filename string) (*cacheSpec, error) {
+	specData, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read file %s: %v", filename, err)
 	}
-	winningBidsByBidder["twoImp"][openrtb_ext.BidderAppnexus] = &bid[3]
-	roundedPrices[winningBidsByBidder["twoImp"][openrtb_ext.BidderAppnexus]] = "1.64"
-	bid[4] = pbsOrtbBid{
-		bid: &openrtb.Bid{
-			Price: 7.64,
-			Exp:   900,
-		},
+
+	var spec cacheSpec
+	if err := json.Unmarshal(specData, &spec); err != nil {
+		return nil, fmt.Errorf("Failed to unmarshal JSON from file: %v", err)
 	}
-	winningBidsByBidder["twoImp"][openrtb_ext.BidderRubicon] = &bid[4]
-	roundedPrices[winningBidsByBidder["twoImp"][openrtb_ext.BidderRubicon]] = "7.64"
-	testAuction := &auction{
-		winningBidsByBidder: winningBidsByBidder,
+
+	return &spec, nil
+}
+
+func runCacheSpec(t *testing.T, fileDisplayName string, specData *cacheSpec) {
+	// bid := make([]pbsOrtbBid, 5)
+	var bid *pbsOrtbBid
+	winningBidsByBidder := make(map[string]map[openrtb_ext.BidderName]*pbsOrtbBid)
+	roundedPrices := make(map[*pbsOrtbBid]string)
+	for i, pbsBid := range specData.PbsBids {
+		if _, ok := winningBidsByBidder[pbsBid.Bid.ID]; !ok {
+			winningBidsByBidder[pbsBid.Bid.ID] = make(map[openrtb_ext.BidderName]*pbsOrtbBid)
+		}
+		bid = &pbsOrtbBid{
+			bid:     pbsBid.Bid,
+			bidType: pbsBid.BidType,
+		}
+		winningBidsByBidder[pbsBid.Bid.ID][pbsBid.Bidder] = bid
+		roundedPrices[bid] = strconv.FormatFloat(bid.bid.Price, 'f', 2, 64)
+		// Marshal the bid for the expected cacheables
+		cjson, _ := json.Marshal(bid.bid)
+		specData.ExpectedCacheables[i].Data = cjson
 	}
 	ctx := context.Background()
 	cache := &mockCache{}
 
-	_ = testAuction.doCache(ctx, cache, true, false, bidRequest, 60)
-	json0, _ := json.Marshal(bid[0].bid)
-	json1, _ := json.Marshal(bid[1].bid)
-	json2, _ := json.Marshal(bid[2].bid)
-	json3, _ := json.Marshal(bid[3].bid)
-	json4, _ := json.Marshal(bid[4].bid)
-	cacheables := make([]prebid_cache_client.Cacheable, 5)
-	cacheables[0] = prebid_cache_client.Cacheable{
-		Type:       prebid_cache_client.TypeJSON,
-		TTLSeconds: 360,
-		Data:       json0,
-	}
-	cacheables[1] = prebid_cache_client.Cacheable{
-		Type:       prebid_cache_client.TypeJSON,
-		TTLSeconds: 260,
-		Data:       json1,
-	}
-	cacheables[2] = prebid_cache_client.Cacheable{
-		Type:       prebid_cache_client.TypeJSON,
-		TTLSeconds: 360,
-		Data:       json2,
-	}
-	cacheables[3] = prebid_cache_client.Cacheable{
-		Type:       prebid_cache_client.TypeJSON,
-		TTLSeconds: 0,
-		Data:       json3,
-	}
-	cacheables[4] = prebid_cache_client.Cacheable{
-		Type:       prebid_cache_client.TypeJSON,
-		TTLSeconds: 960,
-		Data:       json4,
+	testAuction := &auction{
+		winningBidsByBidder: winningBidsByBidder,
 	}
+	_ = testAuction.doCache(ctx, cache, true, false, &specData.BidRequest, 60, &specData.DefaultTTLs)
 	found := 0
 
-	for _, cExpected := range cacheables {
+	for _, cExpected := range specData.ExpectedCacheables {
 		for _, cFound := range cache.items {
 			eq := jsonpatch.Equal(cExpected.Data, cFound.Data)
 			if cExpected.TTLSeconds == cFound.TTLSeconds && eq {
@@ -141,14 +108,27 @@ func TestDoCache(t *testing.T) {
 		}
 	}
 
-	if found != 5 {
-		fmt.Printf("Expected:\n%v\n\n", cacheables)
+	if found != len(specData.ExpectedCacheables) {
+		fmt.Printf("Expected:\n%v\n\n", specData.ExpectedCacheables)
 		fmt.Printf("Found:\n%v\n\n", cache.items)
-		t.Errorf("All expected cacheables not found. Expected 5, found %d.", found)
+		t.Errorf("All expected cacheables not found. Expected %d, found %d.", len(specData.ExpectedCacheables), found)
 	}
 
 }
 
+type cacheSpec struct {
+	BidRequest         openrtb.BidRequest              `json:"bidRequest"`
+	PbsBids            []pbsBid                        `json:"pbsBids"`
+	ExpectedCacheables []prebid_cache_client.Cacheable `json:"expectedCacheables"`
+	DefaultTTLs        config.DefaultTTLs              `json:"defaultTTLs"`
+}
+
+type pbsBid struct {
+	Bid     *openrtb.Bid           `json:"bid"`
+	BidType openrtb_ext.BidType    `json:"bidType"`
+	Bidder  openrtb_ext.BidderName `json:"bidder"`
+}
+
 type mockCache struct {
 	items []prebid_cache_client.Cacheable
 }

exchange/cachetest/defaultbanner.json

@@ -0,0 +1,41 @@
+{
+    "bidRequest": {
+		"imp": [
+			{
+				"id":  "oneImp"
+            }
+        ]
+    },
+    "pbsBids": [{
+        "bid":{
+            "id": "oneImp",
+            "price": 7.64,
+            "exp":   600
+        },
+        "bidType": "banner",
+        "bidder": "appnexus"
+    }, {
+        "bid": {
+            "id": "oneImp",
+            "price": 5.64
+        },
+        "bidType": "banner",
+        "bidder": "pubmatic"
+    }],
+    "expectedCacheables": [
+        {
+            "Type": "json",
+            "TTLSeconds": 660
+        }, {
+            "Type": "json",
+            "TTLSeconds": 360
+        }
+    ],
+    "defaultTTLs": {
+        "banner": 300,
+        "video": 3600,
+        "audio": 1800,
+        "native": 300
+    }
+}
+

exchange/cachetest/defaultvideo.json

@@ -0,0 +1,42 @@
+{
+    "bidRequest": {
+		"imp": [
+			{
+				"id":  "oneImp",
+                "exp":   600
+            }, {
+				"id":  "twoImp"
+            }
+        ]
+    },
+    "pbsBids": [{
+        "bid":{
+            "id": "oneImp",
+            "price": 7.64
+        },
+        "bidType": "video",
+        "bidder": "appnexus"
+    }, {
+        "bid": {
+            "id": "twoImp",
+            "price": 5.64
+        },
+        "bidType": "video",
+        "bidder": "pubmatic"
+    }],
+    "expectedCacheables": [
+        {
+            "Type": "json",
+            "TTLSeconds": 660
+        }, {
+            "Type": "json",
+            "TTLSeconds": 3660
+        }
+    ],
+    "defaultTTLs": {
+        "banner": 300,
+        "video": 3600,
+        "audio": 1800,
+        "native": 300
+    }
+}