diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ab-go-sdk/README.md b/ab-go-sdk/README.md new file mode 100644 index 00000000000..7bb34d6b8b1 --- /dev/null +++ b/ab-go-sdk/README.md @@ -0,0 +1,4 @@ +# ab-go-sdk + +use sample see: +https://github.com/ParticleMedia/ab-go-sdk-sample diff --git a/ab-go-sdk/ab.go b/ab-go-sdk/ab.go new file mode 100644 index 00000000000..ca1c832f94b --- /dev/null +++ b/ab-go-sdk/ab.go @@ -0,0 +1,17 @@ +package ab + +type ABService interface { + AB(*ABContext, *ABResult) +} + +var ab *ABServices + +func Init(cfg *ABConfig) { + if ab == nil { + ab = NewABServices(cfg) + } +} + +func AB(ctx *ABContext) *ABResult { + return ab.AB(ctx) +} diff --git a/ab-go-sdk/ab_config.go b/ab-go-sdk/ab_config.go new file mode 100644 index 00000000000..f604b5ab2aa --- /dev/null +++ b/ab-go-sdk/ab_config.go @@ -0,0 +1,8 @@ +package ab + +type ABConfig struct { + App string + Url string + Layers []string + EnableCohort bool +} diff --git a/ab-go-sdk/ab_context.go b/ab-go-sdk/ab_context.go new file mode 100644 index 00000000000..656d06b2fd5 --- /dev/null +++ b/ab-go-sdk/ab_context.go @@ -0,0 +1,35 @@ +package ab + +import "fmt" + +type ABContext struct { + Factor string + ConditionCtx map[string]interface{} + Userid uint32 +} + +func NewABContext(factor string) *ABContext { + return &ABContext{Factor: factor, ConditionCtx: map[string]interface{}{}} +} + +func (this *ABContext) WithUserid(userid uint32) *ABContext { + this.Userid = userid + this.ConditionCtx["uid"] = userid + return this +} + +func (this *ABContext) WithConditions(conditions map[string]interface{}) *ABContext { + for k, v := range conditions { + this.WithCondition(k, v) + } + return this +} + +func (this *ABContext) WithCondition(key string, value interface{}) *ABContext { + if key == "uid" { + fmt.Println("Cannot Set userid use WithCondition(s), Please use WithUserid") + return this + } + this.ConditionCtx[key] = value + return this +} diff --git a/ab-go-sdk/ab_result.go b/ab-go-sdk/ab_result.go new file mode 100644 index 00000000000..c83e05d3a3a --- /dev/null +++ b/ab-go-sdk/ab_result.go @@ -0,0 +1,42 @@ +package ab + +type ABResult struct { + Exp map[string]string + Config map[string]string + layers map[string]bool +} + +func NewABResult() *ABResult { + return &ABResult{ + Exp: map[string]string{}, + Config: map[string]string{}, + layers: map[string]bool{}, + } +} + +func (this *ABResult) MergeVersion(version *Version) { + if version == nil { + return + } + if version.Exp == nil || this.layers[version.Exp.Layer.Name] { + return + } + this.layers[version.Exp.Layer.Name] = true + this.Exp[version.Exp.Name] = version.Name + for k, v := range version.Config { + _, exists := this.Config[k] + if !exists { + this.Config[k] = v + } + } +} + +func (this *ABResult) MergeVersions(versions []*Version) { + for _, version := range versions { + this.MergeVersion(version) + } +} + +func (this *ABResult) ContainsLayer(layerName string) bool { + return this.layers[layerName] +} diff --git a/ab-go-sdk/ab_services.go b/ab-go-sdk/ab_services.go new file mode 100644 index 00000000000..d9a7352b66a --- /dev/null +++ b/ab-go-sdk/ab_services.go @@ -0,0 +1,109 @@ +package ab + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/golang/glog" + "github.com/robfig/cron/v3" +) + +type ABServices struct { + Cfg *ABConfig + status string + cron *cron.Cron + services []ABService + cohortService *CohortService +} + +func NewABServices(cfg *ABConfig) *ABServices { + s := &ABServices{ + Cfg: cfg, + status: "success", + } + if cfg.EnableCohort { + s.cohortService = NewCohortService(cfg.Url) + } + if err := s.reload(); err != nil { + panic(err) + } + s.cron = cron.New() + // nolint + s.cron.AddFunc("* * * * *", s.Reload) + s.cron.Start() + + return s +} + +func (s *ABServices) Reload() { + start := time.Now() + if err := s.reload(); err != nil { + glog.Errorf("Reload AB config err: %s", err) + s.status = "failed" + return + } + s.status = "success" + glog.Info("Reload AB config success, cost: ", time.Since(start)) +} + +type Layers []*Layer + +func (s Layers) Len() int { + return len(s) +} +func (s Layers) Less(i, j int) bool { + return s[i].Zone < s[j].Zone +} +func (s Layers) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s *ABServices) reload() error { + resp, err := http.Get(s.Cfg.Url + "/ab/newsbreak/layers/" + strings.Join(s.Cfg.Layers, ",") + "?" + url.Values{ + "app": {s.Cfg.App}, + "last_status": {s.status}, + "client_version": {"go-1.0.0"}, + "cohort": {fmt.Sprint(s.Cfg.EnableCohort)}, + }.Encode()) + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + var layers []*Layer + if err = json.Unmarshal(body, &layers); err != nil { + return err + } + if s.cohortService != nil { + s.cohortService.Switching() + } + sort.Sort(Layers(layers)) + for _, layer := range layers { + if err = layer.Init(s.cohortService); err != nil { + return err + } + } + s.services = []ABService{NewUserService(layers), NewBucketService(layers)} + if s.cohortService != nil { + s.cohortService.Switched() + } + + return nil +} + +func (s *ABServices) AB(ctx *ABContext) *ABResult { + result := NewABResult() + for _, service := range s.services { + service.AB(ctx, result) + } + return result +} diff --git a/ab-go-sdk/bucket_service.go b/ab-go-sdk/bucket_service.go new file mode 100644 index 00000000000..312aed0db9e --- /dev/null +++ b/ab-go-sdk/bucket_service.go @@ -0,0 +1,75 @@ +package ab + +import ( + "strings" + + "github.com/antonmedv/expr" + "github.com/spaolacci/murmur3" +) + +type BucketService struct { + service *ZoneLayerService +} + +func NewBucketService(layers []*Layer) *BucketService { + service := NewZoneLayerService() + layersIndex := map[string]*ZoneLayerService{} + for _, layer := range layers { + zone := layer.Zone + zoneLayer := layersIndex[zone] + if zoneLayer == nil { + zoneLayer = NewZoneLayerService() + layersIndex[zone] = zoneLayer + var parentZoneLayer *ZoneLayerService + zones := strings.Split(zone, ".") + for i := len(zones) - 1; i > 0 && parentZoneLayer == nil; i-- { + parentZoneLayer = layersIndex[strings.Join(zones[:i], ".")] + } + if parentZoneLayer != nil { + if parentZoneLayer.Left == nil { + parentZoneLayer.Left = NewZoneLayerService() + } + parentZoneLayer.Left.Selfs = append(parentZoneLayer.Left.Selfs, zoneLayer) + } else { + service.Selfs = append(service.Selfs, zoneLayer) + } + } + zoneLayer.Selfs = append(zoneLayer.Selfs, layer) + } + return &BucketService{service: service} +} + +func (this *BucketService) AB(ctx *ABContext, result *ABResult) { + this.service.ForEachLayer(func(layer *Layer) bool { + if result.ContainsLayer(layer.Name) { + return true + } + platform, _ := ctx.ConditionCtx["platform"].(string) + if layer.Platform != "" && layer.Platform != platform { + return false + } + n := murmur3.Sum32([]byte(layer.shufflePrefix + "@" + ctx.Factor)) + version := layer.GetBucket(n) + if version == nil { + return false + } + if version.Exp.Conditions != "" { + if ctx.ConditionCtx == nil { + return false + } + out, err := expr.Eval(version.Exp.Conditions, ctx.ConditionCtx) + conditionSuccess, conditionBool := out.(bool) + if err != nil || !conditionSuccess || !conditionBool { + return false + } + } + cohort := version.Exp.Cohort + if cohort != nil { + if ctx.Userid == 0 || !cohort.Contains(ctx.Userid) { + return false + } + } + result.MergeVersion(version) + return true + }) +} diff --git a/ab-go-sdk/cohort.go b/ab-go-sdk/cohort.go new file mode 100644 index 00000000000..16703282806 --- /dev/null +++ b/ab-go-sdk/cohort.go @@ -0,0 +1,15 @@ +package ab + +import ( + "github.com/RoaringBitmap/roaring" +) + +type Cohort struct { + Location string `json:"location"` + Users *roaring.Bitmap `json:"-"` + UpdateTime string `json:"update_time"` +} + +func (this *Cohort) Contains(x uint32) bool { + return this.Users != nil && this.Users.Contains(x) +} diff --git a/ab-go-sdk/cohort_service.go b/ab-go-sdk/cohort_service.go new file mode 100644 index 00000000000..7425d3a62ef --- /dev/null +++ b/ab-go-sdk/cohort_service.go @@ -0,0 +1,115 @@ +package ab + +import ( + "bufio" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strconv" + + "github.com/RoaringBitmap/roaring" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/golang/glog" +) + +type CohortService struct { + Current map[string]*Cohort + NG map[string]*Cohort + url string +} + +func NewCohortService(url string) *CohortService { + return &CohortService{ + Current: map[string]*Cohort{}, + NG: map[string]*Cohort{}, + url: url, + } +} + +func (u *CohortService) Switching() { + u.NG = map[string]*Cohort{} +} + +func (u *CohortService) Switched() { + u.Current = u.NG + u.NG = map[string]*Cohort{} +} + +func (u *CohortService) GetCohort(input *Cohort) (tag *Cohort, err error) { + if input == nil { + return nil, nil + } + location := input.Location + update := input.UpdateTime + if tag = u.NG[location]; tag != nil && tag.UpdateTime >= update { + return tag, nil + } + if tag = u.Current[location]; tag != nil && tag.UpdateTime >= update { + u.NG[location] = tag + return tag, nil + } + tag = &Cohort{ + Location: location, + Users: roaring.New(), + UpdateTime: update, + } + if err := u.setupAwsEnv(); err != nil { + return nil, err + } + sess, err := session.NewSession(&aws.Config{}) + if err != nil { + return nil, err + } + svc := s3.New(sess) + var parsedUrl *url.URL + if parsedUrl, err = url.Parse(location); err != nil { + return nil, err + } + s3Object, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(parsedUrl.Host), + Key: aws.String(parsedUrl.Path), + }) + if err != nil { + return nil, fmt.Errorf("Get Cohort: %s, err: %s", location, err) + } + defer s3Object.Body.Close() + scanner := bufio.NewScanner(s3Object.Body) + for scanner.Scan() { + userID, err := strconv.Atoi(scanner.Text()) + if err != nil { + return nil, err + } + tag.Users.AddInt(userID) + } + if err = scanner.Err(); err != nil { + return nil, err + } + u.NG[location] = tag + glog.Info("Reload Cohort success, location: " + location + ", update_time: " + update) + return tag, nil +} + +func (this *CohortService) setupAwsEnv() error { + resp, err := http.Get(this.url + "/sys/aws_envs") + if err != nil { + return err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + envs := map[string]string{} + if err = json.Unmarshal(body, &envs); err != nil { + return err + } + for k, v := range envs { + os.Setenv(k, v) + } + return nil +} diff --git a/ab-go-sdk/exp.go b/ab-go-sdk/exp.go new file mode 100644 index 00000000000..528f06853b0 --- /dev/null +++ b/ab-go-sdk/exp.go @@ -0,0 +1,9 @@ +package ab + +type Exp struct { + Name string `json:"name"` + Versions []*Version `json:"versions"` + Conditions string `json:"conditions"` + Layer *Layer `json:"-"` + Cohort *Cohort `json:"cohort"` +} diff --git a/ab-go-sdk/go.mod b/ab-go-sdk/go.mod new file mode 100644 index 00000000000..d9fc2d354a0 --- /dev/null +++ b/ab-go-sdk/go.mod @@ -0,0 +1,19 @@ +module github.com/ParticleMedia/ab-go-sdk + +go 1.17 + +require ( + github.com/RoaringBitmap/roaring v1.2.3 + github.com/antonmedv/expr v1.12.1 + github.com/aws/aws-sdk-go v1.44.217 + github.com/golang/glog v1.0.0 + github.com/robfig/cron/v3 v3.0.1 + github.com/spaolacci/murmur3 v1.1.0 +) + +require ( + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect +) diff --git a/ab-go-sdk/go.sum b/ab-go-sdk/go.sum new file mode 100644 index 00000000000..405a9e54d50 --- /dev/null +++ b/ab-go-sdk/go.sum @@ -0,0 +1,68 @@ +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/antonmedv/expr v1.12.1 h1:GTGrGN1kxxb+le0uQKaFRK8By4cvq1sleUCGE/U6hHg= +github.com/antonmedv/expr v1.12.1/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= +github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= +github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ab-go-sdk/layer.go b/ab-go-sdk/layer.go new file mode 100644 index 00000000000..20306937b78 --- /dev/null +++ b/ab-go-sdk/layer.go @@ -0,0 +1,63 @@ +package ab + +import "strconv" + +const ( + bucketSize = 1000 +) + +type Layer struct { + Name string `json:"name"` + Platform string `json:"platform"` + Exps []*Exp `json:"exps"` + ShuffleTs string `json:"shuffle_ts"` + shufflePrefix string + BucketsMap map[string]string `json:"buckets"` + Zone string `json:"zone"` + Buckets []*Version `json:"-"` +} + +func (this *Layer) Init(cohortService *CohortService) error { + this.Buckets = make([]*Version, bucketSize) + this.shufflePrefix = this.Name + this.ShuffleTs + versionMap := map[string]*Version{} + for _, exp := range this.Exps { + if exp == nil { + continue + } + exp.Layer = this + if len(exp.Versions) == 0 { + continue + } + if cohortService != nil { + var err error + exp.Cohort, err = cohortService.GetCohort(exp.Cohort) + if err != nil { + return err + } + } + for _, version := range exp.Versions { + version.Exp = exp + versionMap[exp.Name+"-"+version.Name] = version + } + } + for bucket, versionName := range this.BucketsMap { + version := versionMap[versionName] + n, err := strconv.Atoi(bucket) + if err != nil { + continue + } + if n >= 0 && n < bucketSize && version != nil { + this.Buckets[n] = version + } + } + return nil +} + +func (this *Layer) ForEachLayer(action func(*Layer) bool) bool { + return action(this) +} + +func (this *Layer) GetBucket(n uint32) *Version { + return this.Buckets[n%bucketSize] +} diff --git a/ab-go-sdk/layer_service.go b/ab-go-sdk/layer_service.go new file mode 100644 index 00000000000..116bddd7063 --- /dev/null +++ b/ab-go-sdk/layer_service.go @@ -0,0 +1,5 @@ +package ab + +type LayerService interface { + ForEachLayer(func(*Layer) bool) bool +} diff --git a/ab-go-sdk/user_service.go b/ab-go-sdk/user_service.go new file mode 100644 index 00000000000..c9f399b11ba --- /dev/null +++ b/ab-go-sdk/user_service.go @@ -0,0 +1,43 @@ +package ab + +import "strings" + +type UserService struct { + users map[string][]*Version +} + +func NewUserService(layers []*Layer) *UserService { + users := map[string][]*Version{} + userZones := map[string]bool{} + containsZone := func(user string, zone string) bool { + if zone == "" { + return false + } + zones := strings.Split(zone, ".") + for i := len(zones) - 1; i > 0; i-- { + parentZone := strings.Join(zones[:i], ".") + key := user + "@" + parentZone + if userZones[key] { + return true + } + userZones[key] = true + } + return false + } + for _, layer := range layers { + for _, exp := range layer.Exps { + for _, version := range exp.Versions { + for _, user := range version.Users { + if !containsZone(user, layer.Zone) { + users[user] = append(users[user], version) + } + } + } + } + } + return &UserService{users: users} +} + +func (this *UserService) AB(ctx *ABContext, result *ABResult) { + result.MergeVersions(this.users[ctx.Factor]) +} diff --git a/ab-go-sdk/version.go b/ab-go-sdk/version.go new file mode 100644 index 00000000000..03b0ddbb096 --- /dev/null +++ b/ab-go-sdk/version.go @@ -0,0 +1,8 @@ +package ab + +type Version struct { + Name string `json:"name"` + Config map[string]string `json:"config"` + Users []string `json:"users"` + Exp *Exp `json:"-"` +} diff --git a/ab-go-sdk/zone_layer_service.go b/ab-go-sdk/zone_layer_service.go new file mode 100644 index 00000000000..516a29189c6 --- /dev/null +++ b/ab-go-sdk/zone_layer_service.go @@ -0,0 +1,18 @@ +package ab + +type ZoneLayerService struct { + Left *ZoneLayerService + Selfs []LayerService +} + +func NewZoneLayerService() *ZoneLayerService { + return &ZoneLayerService{} +} + +func (this *ZoneLayerService) ForEachLayer(action func(layer *Layer) bool) bool { + hit := false + for _, self := range this.Selfs { + hit = self.ForEachLayer(action) || hit + } + return hit || this.Left != nil && this.Left.ForEachLayer(action) +} diff --git a/adapters/applovin/applovin.go b/adapters/applovin/applovin.go new file mode 100644 index 00000000000..084b9d26beb --- /dev/null +++ b/adapters/applovin/applovin.go @@ -0,0 +1,148 @@ +package applovin + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/prebid/openrtb/v17/openrtb2" + "github.com/prebid/prebid-server/adapters" + "github.com/prebid/prebid-server/config" + "github.com/prebid/prebid-server/errortypes" + "github.com/prebid/prebid-server/openrtb_ext" +) + +type ApplovinAdapter struct { + endpoint string + token string +} + +func (a *ApplovinAdapter) MakeRequests(request *openrtb2.BidRequest, _ *adapters.ExtraRequestInfo) ([]*adapters.RequestData, []error) { + headers := http.Header{} + headers.Add("Content-Type", "application/json;charset=utf-8") + headers.Add("Accept", "application/json") + headers.Add("X-Account-Key", a.token) + impressions := request.Imp + result := make([]*adapters.RequestData, 0, len(impressions)) + errs := make([]error, 0, len(impressions)) + + for _, impression := range impressions { + if impression.Banner == nil && impression.Video == nil && impression.Native == nil { + errs = append(errs, &errortypes.BadInput{ + Message: "Applovin only supports banner, video or native ads", + }) + continue + } + if impression.Banner != nil { + if impression.Banner.W == nil || impression.Banner.H == nil || *impression.Banner.W == 0 || *impression.Banner.H == 0 { + if len(impression.Banner.Format) == 0 { + errs = append(errs, &errortypes.BadInput{ + Message: "banner size information missing", + }) + continue + } + banner := *impression.Banner + banner.W = openrtb2.Int64Ptr(banner.Format[0].W) + banner.H = openrtb2.Int64Ptr(banner.Format[0].H) + impression.Banner = &banner + } + } + if len(impression.Ext) == 0 { + errs = append(errs, errors.New("impression extensions required")) + continue + } + var bidderExt adapters.ExtImpBidder + err := json.Unmarshal(impression.Ext, &bidderExt) + if err != nil { + errs = append(errs, err) + continue + } + if len(bidderExt.Bidder) == 0 { + errs = append(errs, errors.New("bidder required")) + continue + } + var impressionExt openrtb_ext.ExtImpApplovin + err = json.Unmarshal(bidderExt.Bidder, &impressionExt) + if err != nil { + errs = append(errs, err) + continue + } + if impressionExt.TagID == "" { + errs = append(errs, errors.New("Applovin token required")) + continue + } + request.Imp = []openrtb2.Imp{impression} + body, err := json.Marshal(request) + if err != nil { + errs = append(errs, err) + continue + } + result = append(result, &adapters.RequestData{ + Method: "POST", + Uri: a.endpoint, + Body: body, + Headers: headers, + }) + } + + request.Imp = impressions + + if len(result) == 0 { + return nil, errs + } + return result, errs +} + +func (a *ApplovinAdapter) MakeBids(bidReq *openrtb2.BidRequest, adapterReq *adapters.RequestData, adapterResp *adapters.ResponseData) (*adapters.BidderResponse, []error) { + if adapterResp.StatusCode != http.StatusOK { + if adapterResp.StatusCode == http.StatusNoContent { + return nil, nil + } + if adapterResp.StatusCode == http.StatusBadRequest { + return nil, []error{&errortypes.BadInput{ + Message: fmt.Sprintf("Unexpected status code: %d", adapterResp.StatusCode), + }} + } + return nil, []error{&errortypes.BadServerResponse{ + Message: fmt.Sprintf("Unexpected status code: %d", adapterResp.StatusCode), + }} + } + + var bidResp openrtb2.BidResponse + if err := json.Unmarshal(adapterResp.Body, &bidResp); err != nil { + return nil, []error{&errortypes.BadServerResponse{ + Message: fmt.Sprintf("Failed to unmarshal bid response: %s", err.Error()), + }} + } + + bidderResp := adapters.NewBidderResponseWithBidsCapacity(len(bidReq.Imp)) + var errors []error + + for _, seatbid := range bidResp.SeatBid { + for _, bid := range seatbid.Bid { + bid.ImpID = bidReq.Imp[0].ID + bidderResp.Bids = append(bidderResp.Bids, &adapters.TypedBid{ + Bid: &bid, + BidType: openrtb_ext.BidTypeBanner, + }) + break + } + } + + if bidResp.Cur != "" { + bidderResp.Currency = bidResp.Cur + } + return bidderResp, errors +} + +func Builder(bidderName openrtb_ext.BidderName, config config.Adapter, server config.Server) (adapters.Bidder, error) { + if config.AppSecret == "" { + return nil, errors.New("AppSecret is not configured. Did you set adapters.applovin.app_secret in the app config?") + } + bidder := &ApplovinAdapter{ + endpoint: config.Endpoint, + token: config.AppSecret, + } + return bidder, nil +} diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index addb2c1d462..e85c08bc05e 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -15,6 +15,7 @@ import ( "github.com/golang/glog" "github.com/julienschmidt/httprouter" "github.com/prebid/openrtb/v17/openrtb2" + "github.com/prebid/prebid-server/util/task" "github.com/prebid/prebid-server/util/uuidutil" jsonpatch "gopkg.in/evanphx/json-patch.v4" @@ -58,6 +59,7 @@ func NewAmpEndpoint( defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName, storedRespFetcher stored_requests.Fetcher, + ab *task.AB, ) (httprouter.Handle, error) { if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { @@ -88,7 +90,8 @@ func NewAmpEndpoint( nil, nil, ipValidator, - storedRespFetcher}).AmpAuction), nil + storedRespFetcher, + ab}).AmpAuction), nil } diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go index 96252c8bac5..2bb4a8aea8d 100644 --- a/endpoints/openrtb2/amp_auction_test.go +++ b/endpoints/openrtb2/amp_auction_test.go @@ -210,6 +210,7 @@ func TestAMPPageInfo(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) request := httptest.NewRequest("GET", fmt.Sprintf("/openrtb2/auction/amp?tag_id=1&curl=%s", url.QueryEscape(page)), nil) recorder := httptest.NewRecorder() @@ -312,6 +313,7 @@ func TestGDPRConsent(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) // Invoke Endpoint @@ -697,6 +699,7 @@ func TestCCPAConsent(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) // Invoke Endpoint @@ -809,6 +812,7 @@ func TestConsentWarnings(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) // Invoke Endpoint @@ -906,6 +910,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) // Invoke Endpoint @@ -959,6 +964,7 @@ func TestAMPSiteExt(t *testing.T) { nil, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) request, err := http.NewRequest("GET", "/openrtb2/auction/amp?tag_id=1", nil) if !assert.NoError(t, err) { @@ -1000,6 +1006,7 @@ func TestAmpBadRequests(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) for requestID := range badRequests { request := httptest.NewRequest("GET", fmt.Sprintf("/openrtb2/auction/amp?tag_id=%s", requestID), nil) @@ -1032,6 +1039,7 @@ func TestAmpDebug(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) for requestID := range requests { @@ -1108,6 +1116,7 @@ func TestQueryParamOverrides(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) requestID := "1" @@ -1264,6 +1273,7 @@ func (s formatOverrideSpec) execute(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) url := fmt.Sprintf("/openrtb2/auction/amp?tag_id=1&debug=1&w=%d&h=%d&ow=%d&oh=%d&ms=%s&account=%s", s.width, s.height, s.overrideWidth, s.overrideHeight, s.multisize, s.account) @@ -1723,6 +1733,7 @@ func ampObjectTestSetup(t *testing.T, inTagId string, inStoredRequest json.RawMe []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) return &actualAmpObject, endpoint } @@ -1774,6 +1785,7 @@ func TestAmpAuctionResponseHeaders(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) for _, test := range testCases { @@ -1808,6 +1820,7 @@ func TestRequestWithTargeting(t *testing.T) { nil, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) url, err := url.Parse("/openrtb2/auction/amp") assert.NoError(t, err, "unexpected error received while parsing url") diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index ca1a4679612..b925002e0b9 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -42,6 +42,7 @@ import ( "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/httputil" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/task" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" ) @@ -70,6 +71,7 @@ func NewEndpoint( defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName, storedRespFetcher stored_requests.Fetcher, + ab *task.AB, ) (httprouter.Handle, error) { if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { return nil, errors.New("NewEndpoint requires non-nil arguments.") @@ -99,7 +101,8 @@ func NewEndpoint( nil, nil, ipValidator, - storedRespFetcher}).Auction), nil + storedRespFetcher, + ab}).Auction), nil } type endpointDeps struct { @@ -120,6 +123,34 @@ type endpointDeps struct { debugLogRegexp *regexp.Regexp privateNetworkIPValidator iputil.IPValidator storedRespFetcher stored_requests.Fetcher + ab *task.AB +} + +type LogIno struct { + Os string + Idfa string + Country string + City string + Region string + AdUnit string + Fill bool + Uid string + Price float64 + Lat float64 + Lon float64 + AppVersion string + ReqId string + SkadnSize int + HasBuyerId bool + AbBuckets []string +} + +type SkadnExt struct { + Skadn SkadnData `json:"skadn"` +} + +type SkadnData struct { + SkadnList []string `json:"skadnetids"` } func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { @@ -143,16 +174,93 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http PubID: metrics.PublisherUnknown, CookieFlag: metrics.CookieFlagUnknown, RequestStatus: metrics.RequestStatusOK, + StoredImp: metrics.StoredImpUnknown, + AbBuckets: []string{metrics.AbBucketUnknown}, + } + + w.Header().Set("X-Prebid", version.BuildXPrebidHeader(version.Ver)) + + logMsg := LogIno{ + Os: metrics.LogUnknown, + Idfa: metrics.LogUnknown, + Country: metrics.LogUnknown, + City: metrics.LogUnknown, + Region: metrics.LogUnknown, + AdUnit: metrics.LogUnknown, + Fill: false, + Uid: metrics.LogUnknown, + Price: -1, + Lat: 0, + Lon: 0, + AppVersion: metrics.LogUnknown, + SkadnSize: 0, + ReqId: metrics.LogUnknown, + HasBuyerId: false, + AbBuckets: []string{metrics.AbBucketUnknown}, + } + + req, impExtInfoMap, storedAuctionResponses, storedBidResponses, bidderImpReplaceImp, storedImp, errL := deps.parseRequest(r, &labels, &logMsg) + + if len(req.Imp) > 0 { + logMsg.ReqId = req.ID + skadn := &SkadnExt{} + err := json.Unmarshal(req.Imp[0].Ext, skadn) + if err == nil { + logMsg.SkadnSize = len(skadn.Skadn.SkadnList) + } + } + + adUnitName := "unknown" + if storedImp != nil { + if string(storedImp) != "" { + adUnitName = string(storedImp) + logMsg.AdUnit = adUnitName + } + } + labels.StoredImp = adUnitName + + if req.Device != nil { + if req.Device.OS != "" { + logMsg.Os = req.Device.OS + // hardcode ios bundle + if req.Device.OS == "iOS" || req.Device.OS == "iPadOS" { + req.App.Bundle = "1132762804" + } + } + + if req.Device.IFA != "" { + logMsg.Idfa = req.Device.IFA + } + + if req.Device.Geo != nil { + logMsg.Country = req.Device.Geo.Country + logMsg.City = req.Device.Geo.City + logMsg.Region = req.Device.Geo.Region + logMsg.Lat = req.Device.Geo.Lat + logMsg.Lon = req.Device.Geo.Lon + } + } + + if req.App != nil { + if req.App.Ver != "" { + logMsg.AppVersion = req.App.Ver + } } + + if req.User != nil { + logMsg.Uid = req.User.ID + if req.User.BuyerUID != "" { + logMsg.HasBuyerId = true + } + } + defer func() { deps.metricsEngine.RecordRequest(labels) deps.metricsEngine.RecordRequestTime(labels, time.Since(start)) deps.analytics.LogAuctionObject(&ao) + writeLogInfo(logMsg) }() - w.Header().Set("X-Prebid", version.BuildXPrebidHeader(version.Ver)) - - req, impExtInfoMap, storedAuctionResponses, storedBidResponses, bidderImpReplaceImp, errL := deps.parseRequest(r) if errortypes.ContainsFatalError(errL) && writeError(errL, w, &labels) { return } @@ -214,11 +322,24 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http StoredBidResponses: storedBidResponses, BidderImpReplaceImpID: bidderImpReplaceImp, PubID: labels.PubID, + StoredImp: adUnitName, + AbBucketList: labels.AbBuckets, } + response, err := deps.ex.HoldAuction(ctx, auctionRequest, nil) ao.Request = req.BidRequest ao.Response = response ao.Account = account + + if response != nil && response.SeatBid != nil { + for _, seatBid := range response.SeatBid { + for _, bid := range seatBid.Bid { + logMsg.Fill = true + logMsg.Price = bid.Price + } + } + } + if err != nil { if errortypes.ReadCode(err) == errortypes.BadInputErrorCode { writeError([]error{err}, w, &labels) @@ -248,6 +369,14 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http } } +func writeLogInfo(logMsg LogIno) { + out, err := json.Marshal(logMsg) + if err != nil { + panic(err) + } + fmt.Println(string(out)) +} + // parseRequest turns the HTTP request into an OpenRTB request. This is guaranteed to return: // // - A context which times out appropriately, given the request. @@ -258,7 +387,7 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http // possible, it will return errors with messages that suggest improvements. // // If the errors list has at least one element, then no guarantees are made about the returned request. -func (deps *endpointDeps) parseRequest(httpRequest *http.Request) (req *openrtb_ext.RequestWrapper, impExtInfoMap map[string]exchange.ImpExtInfo, storedAuctionResponses stored_responses.ImpsWithBidResponses, storedBidResponses stored_responses.ImpBidderStoredResp, bidderImpReplaceImpId stored_responses.BidderImpReplaceImpID, errs []error) { +func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metrics.Labels, logMsg *LogIno) (req *openrtb_ext.RequestWrapper, impExtInfoMap map[string]exchange.ImpExtInfo, storedAuctionResponses stored_responses.ImpsWithBidResponses, storedBidResponses stored_responses.ImpBidderStoredResp, bidderImpReplaceImpId stored_responses.BidderImpReplaceImpID, storedImp []byte, errs []error) { req = &openrtb_ext.RequestWrapper{} req.BidRequest = &openrtb2.BidRequest{} errs = nil @@ -287,18 +416,18 @@ func (deps *endpointDeps) parseRequest(httpRequest *http.Request) (req *openrtb_ impInfo, errs := parseImpInfo(requestJson) if len(errs) > 0 { - return nil, nil, nil, nil, nil, errs + return nil, nil, nil, nil, nil, nil, errs } // Fetch the Stored Request data and merge it into the HTTP request. - if requestJson, impExtInfoMap, errs = deps.processStoredRequests(ctx, requestJson, impInfo); len(errs) > 0 { + if requestJson, impExtInfoMap, storedImp, errs = deps.processStoredRequests(ctx, requestJson, impInfo, labels, logMsg); len(errs) > 0 { return } //Stored auction responses should be processed after stored requests due to possible impression modification storedAuctionResponses, storedBidResponses, bidderImpReplaceImpId, errs = stored_responses.ProcessStoredResponses(ctx, requestJson, deps.storedRespFetcher, deps.bidderMap) if len(errs) > 0 { - return nil, nil, nil, nil, nil, errs + return nil, nil, nil, nil, nil, nil, errs } if err := json.Unmarshal(requestJson, req.BidRequest); err != nil { @@ -1611,11 +1740,11 @@ func getJsonSyntaxError(testJSON []byte) (bool, string) { return false, "" } -func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson []byte, impInfo []ImpExtPrebidData) ([]byte, map[string]exchange.ImpExtInfo, []error) { +func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson []byte, impInfo []ImpExtPrebidData, labels *metrics.Labels, logMsg *LogIno) ([]byte, map[string]exchange.ImpExtInfo, []byte, []error) { // Parse the Stored Request IDs from the BidRequest and Imps. storedBidRequestId, hasStoredBidRequest, err := getStoredRequestId(requestJson) if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } // Fetch the Stored Request data @@ -1639,11 +1768,11 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson storedRequests, storedImps, errs := deps.storedReqFetcher.FetchRequests(ctx, storedReqIds, impStoredReqIds) if len(errs) != 0 { - return nil, nil, errs + return nil, nil, nil, errs } bidRequestID, err := getBidRequestID(storedRequests[storedBidRequestId]) if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } // Apply the Stored BidRequest, if it exists @@ -1652,28 +1781,28 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson if hasStoredBidRequest { isAppRequest, err := checkIfAppRequest(requestJson) if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } if (deps.cfg.GenerateRequestID && isAppRequest) || bidRequestID == "{{UUID}}" { uuidPatch, err := generateUuidForBidRequest(deps.uuidGenerator) if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } uuidPatch, err = jsonpatch.MergePatch(storedRequests[storedBidRequestId], uuidPatch) if err != nil { errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId) - return nil, nil, errL + return nil, nil, nil, errL } resolvedRequest, err = jsonpatch.MergePatch(requestJson, uuidPatch) if err != nil { errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId) - return nil, nil, errL + return nil, nil, nil, errL } } else { resolvedRequest, err = jsonpatch.MergePatch(storedRequests[storedBidRequestId], requestJson) if err != nil { errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId) - return nil, nil, errL + return nil, nil, nil, errL } } } @@ -1691,7 +1820,7 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson err = fmt.Errorf("Invalid JSON in Default Request Settings: %s", Err) } } - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } resolvedRequest = aliasedRequest } @@ -1715,12 +1844,12 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson err = fmt.Errorf("imp.ext.prebid.storedrequest.id %s: Stored Imp has Invalid JSON: %s", impData.ImpExtPrebid.StoredRequest.ID, errMessage) } } - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } resolvedImps = append(resolvedImps, resolvedImp) impId, err := jsonparser.GetString(resolvedImp, "id") if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } echoVideoAttributes := false @@ -1731,7 +1860,7 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson // Extract Passthrough from Merged Imp passthrough, _, _, err := jsonparser.Get(resolvedImp, "ext", "prebid", "passthrough") if err != nil && err != jsonparser.KeyPathNotFoundError { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } impExtInfoMap[impId] = exchange.ImpExtInfo{EchoVideoAttrs: echoVideoAttributes, StoredImp: storedImps[impData.ImpExtPrebid.StoredRequest.ID], Passthrough: passthrough} @@ -1742,7 +1871,7 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson if err == jsonparser.KeyPathNotFoundError { err = fmt.Errorf("request.imp[%d] missing required field: \"id\"\n", i) } - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } impExtInfoMap[impId] = exchange.ImpExtInfo{Passthrough: impData.ImpExtPrebid.Passthrough} } @@ -1750,15 +1879,53 @@ func (deps *endpointDeps) processStoredRequests(ctx context.Context, requestJson if len(resolvedImps) > 0 { newImpJson, err := json.Marshal(resolvedImps) if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} } resolvedRequest, err = jsonparser.Set(resolvedRequest, newImpJson, "imp") if err != nil { - return nil, nil, []error{err} + return nil, nil, nil, []error{err} + } + } + + if deps.ab != nil { + req := &openrtb2.BidRequest{} + if err := json.Unmarshal(resolvedRequest, req); err != nil { + return nil, nil, nil, []error{err} } + + if req.Device != nil && req.App != nil { + err, bucketList := deps.ab.GetBucket(req.User.ID, req.Device.OS, req.App.Ver) + if err != nil { + return nil, nil, nil, []error{err} + } + labels.AbBuckets = nil + logMsg.AbBuckets = nil + + storedABs, _ := deps.storedReqFetcher.FetchABs(ctx, bucketList) + + for _, bucketName := range bucketList { + // only use configed experiments on server + val, ok := storedABs[bucketName] + if ok { + if len(labels.AbBuckets) > 0 && bucketName == "all" { + continue + } + resolvedRequest, err = jsonpatch.MergePatch(resolvedRequest, val) + labels.AbBuckets = append(labels.AbBuckets, bucketName) + logMsg.AbBuckets = append(logMsg.AbBuckets, bucketName) + if err != nil { + return nil, nil, nil, []error{err} + } + } + } + } + } + + if len(impStoredReqIds) == 0 { + return resolvedRequest, impExtInfoMap, nil, nil } - return resolvedRequest, impExtInfoMap, nil + return resolvedRequest, impExtInfoMap, []byte(impStoredReqIds[0]), nil } // parseImpInfo parses the request JSON and returns impression and unmarshalled imp.ext.prebid diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index 95fd784275d..527ba6a8f2c 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -113,6 +113,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { []byte{}, nil, empty_fetcher.EmptyFetcher{}, + nil, ) b.ResetTimer() diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index c92c762466c..dedaced5e32 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -379,7 +379,8 @@ func TestExplicitUserId(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, + nil) endpoint(httptest.NewRecorder(), request, nil) @@ -434,7 +435,7 @@ func doBadAliasRequest(t *testing.T, filename string, expectMsg string) { disabledBidders, aliasJSON, bidderMap, - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) request := httptest.NewRequest("POST", "/openrtb2/auction", bytes.NewReader(testBidRequest)) recorder := httptest.NewRecorder() @@ -485,7 +486,7 @@ func TestNilExchange(t *testing.T) { analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil Exchange.") @@ -508,7 +509,7 @@ func TestNilValidator(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil BidderParamValidator.") @@ -531,7 +532,7 @@ func TestExchangeError(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, "site.json"))) recorder := httptest.NewRecorder() @@ -655,7 +656,7 @@ func TestImplicitIPsEndToEnd(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, test.reqJSONFile))) httpReq.Header.Set("X-Forwarded-For", test.xForwardedForHeader) @@ -852,7 +853,7 @@ func TestImplicitDNTEndToEnd(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, test.reqJSONFile))) httpReq.Header.Set("DNT", test.dntHeader) @@ -1062,6 +1063,7 @@ func TestStoredRequests(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } testStoreVideoAttr := []bool{true, true, false, false, false} @@ -1069,7 +1071,7 @@ func TestStoredRequests(t *testing.T) { for i, requestData := range testStoredRequests { impInfo, errs := parseImpInfo([]byte(requestData)) assert.Len(t, errs, 0, "No errors should be returned") - newRequest, impExtInfoMap, errList := deps.processStoredRequests(context.Background(), json.RawMessage(requestData), impInfo) + newRequest, impExtInfoMap, _, errList := deps.processStoredRequests(context.Background(), json.RawMessage(requestData), impInfo, &metrics.Labels{}, &LogIno{}) if len(errList) != 0 { for _, err := range errList { if err != nil { @@ -1433,6 +1435,7 @@ func TestValidateRequest(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } testCases := []struct { @@ -1805,6 +1808,7 @@ func TestSetIntegrationType(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } testCases := []struct { @@ -1869,6 +1873,7 @@ func TestStoredRequestGenerateUuid(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := &openrtb2.BidRequest{} @@ -1935,7 +1940,7 @@ func TestStoredRequestGenerateUuid(t *testing.T) { deps.cfg.GenerateRequestID = test.givenGenerateRequestID impInfo, errs := parseImpInfo([]byte(test.givenRawData)) assert.Empty(t, errs, test.description) - newRequest, _, errList := deps.processStoredRequests(context.Background(), json.RawMessage(test.givenRawData), impInfo) + newRequest, _, _, errList := deps.processStoredRequests(context.Background(), json.RawMessage(test.givenRawData), impInfo, &metrics.Labels{}, &LogIno{}) assert.Empty(t, errList, test.description) if err := json.Unmarshal(newRequest, req); err != nil { @@ -1969,6 +1974,7 @@ func TestOversizedRequest(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) @@ -2006,6 +2012,7 @@ func TestRequestSizeEdgeCase(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) @@ -2037,6 +2044,7 @@ func TestNoEncoding(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, "site.json"))) recorder := httptest.NewRecorder() @@ -2120,6 +2128,7 @@ func TestContentType(t *testing.T) { []byte{}, openrtb_ext.BuildBidderMap(), empty_fetcher.EmptyFetcher{}, + nil, ) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, "site.json"))) recorder := httptest.NewRecorder() @@ -2339,6 +2348,7 @@ func TestValidateImpExt(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } for _, group := range testGroups { @@ -2390,6 +2400,7 @@ func TestCurrencyTrunc(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -2436,6 +2447,7 @@ func TestCCPAInvalid(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -2486,6 +2498,7 @@ func TestNoSaleInvalid(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -2539,6 +2552,7 @@ func TestValidateSourceTID(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -2582,6 +2596,7 @@ func TestSChainInvalid(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -2933,6 +2948,7 @@ func TestEidPermissionsInvalid(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } ui := int64(1) @@ -3180,7 +3196,7 @@ func TestIOS14EndToEnd(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, "app-ios140-no-ifa.json"))) @@ -3215,6 +3231,7 @@ func TestAuctionWarnings(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) @@ -3256,11 +3273,12 @@ func TestParseRequestParseImpInfoError(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(reqBody)) - resReq, impExtInfoMap, _, _, _, errL := deps.parseRequest(req) + resReq, impExtInfoMap, _, _, _, _, errL := deps.parseRequest(req, &metrics.Labels{}, &LogIno{}) assert.Nil(t, resReq, "Result request should be nil due to incorrect imp") assert.Nil(t, impExtInfoMap, "Impression info map should be nil due to incorrect imp") @@ -3726,7 +3744,7 @@ func TestAuctionResponseHeaders(t *testing.T) { map[string]string{}, []byte{}, openrtb_ext.BuildBidderMap(), - empty_fetcher.EmptyFetcher{}) + empty_fetcher.EmptyFetcher{}, nil) for _, test := range testCases { httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(test.requestBody)) @@ -3829,11 +3847,12 @@ func TestParseRequestMergeBidderParams(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(test.givenRequestBody)) - resReq, _, _, _, _, errL := deps.parseRequest(req) + resReq, _, _, _, _, _, errL := deps.parseRequest(req, &metrics.Labels{}, &LogIno{}) assert.NoError(t, resReq.RebuildRequest()) @@ -3931,11 +3950,12 @@ func TestParseRequestStoredResponses(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, &mockStoredResponseFetcher{mockStoredResponses}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(test.givenRequestBody)) - _, _, storedResponses, _, _, errL := deps.parseRequest(req) + _, _, storedResponses, _, _, _, errL := deps.parseRequest(req, &metrics.Labels{}, &LogIno{}) if test.expectedErrorCount == 0 { assert.Equal(t, test.expectedStoredResponses, storedResponses, "stored responses should match") @@ -4018,10 +4038,11 @@ func TestParseRequestStoredBidResponses(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, &mockStoredResponseFetcher{mockStoredBidResponses}, + nil, } req := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(test.givenRequestBody)) - _, _, _, storedBidResponses, _, errL := deps.parseRequest(req) + _, _, _, storedBidResponses, _, _, errL := deps.parseRequest(req, &metrics.Labels{}, &LogIno{}) if test.expectedErrorCount == 0 { assert.Equal(t, test.expectedStoredBidResponses, storedBidResponses, "stored responses should match") @@ -4051,6 +4072,7 @@ func TestValidateStoredResp(t *testing.T) { nil, hardcodedResponseIPValidator{response: true}, &mockStoredResponseFetcher{}, + nil, } testCases := []struct { @@ -4609,6 +4631,10 @@ func (cf *mockStoredResponseFetcher) FetchRequests(ctx context.Context, requestI return nil, nil, nil } +func (cf *mockStoredResponseFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (cf *mockStoredResponseFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return cf.data, nil } diff --git a/endpoints/openrtb2/test_utils.go b/endpoints/openrtb2/test_utils.go index 02b5a62001d..42321048c86 100644 --- a/endpoints/openrtb2/test_utils.go +++ b/endpoints/openrtb2/test_utils.go @@ -35,6 +35,7 @@ import ( "github.com/prebid/prebid-server/stored_requests" "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/task" "github.com/prebid/prebid-server/util/uuidutil" ) @@ -838,6 +839,10 @@ func (cf mockStoredReqFetcher) FetchResponses(ctx context.Context, ids []string) return nil, nil } +func (cf *mockStoredReqFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + // mockExchange implements the Exchange interface type mockExchange struct { lastRequest *openrtb2.BidRequest @@ -1292,7 +1297,7 @@ func buildTestEndpoint(test testCase, cfg *config.Configuration) (httprouter.Han }, } - var endpointBuilder func(uuidutil.UUIDGenerator, exchange.Exchange, openrtb_ext.BidderParamValidator, stored_requests.Fetcher, stored_requests.AccountFetcher, *config.Configuration, metrics.MetricsEngine, analytics.PBSAnalyticsModule, map[string]string, []byte, map[string]openrtb_ext.BidderName, stored_requests.Fetcher) (httprouter.Handle, error) + var endpointBuilder func(uuidutil.UUIDGenerator, exchange.Exchange, openrtb_ext.BidderParamValidator, stored_requests.Fetcher, stored_requests.AccountFetcher, *config.Configuration, metrics.MetricsEngine, analytics.PBSAnalyticsModule, map[string]string, []byte, map[string]openrtb_ext.BidderName, stored_requests.Fetcher, *task.AB) (httprouter.Handle, error) switch test.endpointType { case AMP_ENDPOINT: @@ -1314,6 +1319,7 @@ func buildTestEndpoint(test testCase, cfg *config.Configuration) (httprouter.Han []byte(test.Config.AliasJSON), bidderMap, storedResponseFetcher, + nil, ) return endpoint, testExchange.(*exchangeTestWrapper), mockBidServersArray, mockCurrencyRatesServer, err @@ -1345,6 +1351,10 @@ func (cf *mockAmpStoredReqFetcher) FetchRequests(ctx context.Context, requestIDs return cf.data, nil, nil } +func (cf *mockAmpStoredReqFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (cf *mockAmpStoredReqFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return nil, nil } @@ -1357,6 +1367,10 @@ func (cf *mockAmpStoredResponseFetcher) FetchRequests(ctx context.Context, reque return nil, nil, nil } +func (cf *mockAmpStoredResponseFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (cf *mockAmpStoredResponseFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { for _, storedResponseID := range ids { if storedResponse, exists := cf.data[storedResponseID]; exists { diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index 0b59c19bb02..0ba6199dc76 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -33,6 +33,7 @@ import ( "github.com/prebid/prebid-server/stored_requests/backends/empty_fetcher" "github.com/prebid/prebid-server/usersync" "github.com/prebid/prebid-server/util/iputil" + "github.com/prebid/prebid-server/util/task" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" ) @@ -53,6 +54,7 @@ func NewVideoEndpoint( defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName, cache prebid_cache_client.Client, + ab *task.AB, ) (httprouter.Handle, error) { if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { @@ -85,7 +87,8 @@ func NewVideoEndpoint( cache, videoEndpointRegexp, ipValidator, - empty_fetcher.EmptyFetcher{}}).VideoAuctionEndpoint), nil + empty_fetcher.EmptyFetcher{}, + ab}).VideoAuctionEndpoint), nil } /* diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index fe1014bf2a9..8f623a12b1a 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -1236,6 +1236,7 @@ func mockDepsWithMetrics(t *testing.T, ex *mockExchangeVideo) (*endpointDeps, *m nil, hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } return deps, metrics, mockModule } @@ -1280,6 +1281,7 @@ func mockDeps(t *testing.T, ex *mockExchangeVideo) *endpointDeps { regexp.MustCompile(`[<>]`), hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } } @@ -1302,6 +1304,7 @@ func mockDepsAppendBidderNames(t *testing.T, ex *mockExchangeAppendBidderNames) regexp.MustCompile(`[<>]`), hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } return deps @@ -1326,6 +1329,7 @@ func mockDepsNoBids(t *testing.T, ex *mockExchangeVideoNoBids) *endpointDeps { regexp.MustCompile(`[<>]`), hardcodedResponseIPValidator{response: true}, empty_fetcher.EmptyFetcher{}, + nil, } return edep @@ -1357,6 +1361,10 @@ func (cf mockVideoStoredReqFetcher) FetchResponses(ctx context.Context, ids []st return nil, nil } +func (cf mockVideoStoredReqFetcher) FetchABs(ctx context.Context, bucketList []string) (data map[string]json.RawMessage, errs []error) { + return nil, nil +} + type mockExchangeVideo struct { lastRequest *openrtb2.BidRequest cache *mockCacheClient diff --git a/exchange/adapter_builders.go b/exchange/adapter_builders.go index 26abc217248..2b47c7d581f 100755 --- a/exchange/adapter_builders.go +++ b/exchange/adapter_builders.go @@ -32,6 +32,7 @@ import ( "github.com/prebid/prebid-server/adapters/amx" "github.com/prebid/prebid-server/adapters/apacdex" "github.com/prebid/prebid-server/adapters/applogy" + "github.com/prebid/prebid-server/adapters/applovin" "github.com/prebid/prebid-server/adapters/appnexus" "github.com/prebid/prebid-server/adapters/appush" "github.com/prebid/prebid-server/adapters/audienceNetwork" @@ -275,6 +276,7 @@ func newAdapterBuilders() map[openrtb_ext.BidderName]adapters.Builder { openrtb_ext.BidderPangle: pangle.Builder, openrtb_ext.BidderPGAM: adtelligent.Builder, openrtb_ext.BidderPubmatic: pubmatic.Builder, + openrtb_ext.BidderApplovin: applovin.Builder, openrtb_ext.BidderPubnative: pubnative.Builder, openrtb_ext.BidderPulsepoint: pulsepoint.Builder, openrtb_ext.BidderQuantumdex: apacdex.Builder, diff --git a/exchange/exchange.go b/exchange/exchange.go index d52f0a33b24..b1f03719fca 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -181,6 +181,8 @@ type AuctionRequest struct { StoredBidResponses stored_responses.ImpBidderStoredResp BidderImpReplaceImpID stored_responses.BidderImpReplaceImpID PubID string + StoredImp string + AbBucketList []string } // BidderRequest holds the bidder specific request and all other @@ -260,6 +262,11 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog * // Slice of BidRequests, each a copy of the original cleaned to only contain bidder data for the named bidder bidderRequests, privacyLabels, errs := cleanOpenRTBRequests(ctx, r, requestExt, e.bidderToSyncerKey, e.me, gdprDefaultValue, e.privacyConfig, e.gdprPermsBuilder, e.tcf2ConfigBuilder, e.hostSChainNode) + for i := range bidderRequests { + bidderRequests[i].BidderLabels.StoredImp = r.StoredImp + bidderRequests[i].BidderLabels.AbBuckets = r.AbBucketList + } + e.me.RecordRequestPrivacy(privacyLabels) if len(r.StoredAuctionResponses) > 0 || len(r.StoredBidResponses) > 0 { @@ -378,6 +385,25 @@ func (e *exchange) HoldAuction(ctx context.Context, r AuctionRequest, debugLog * } } + if auc != nil && auc.winningBidsByBidder != nil { + for impId, topBidsPerImp := range auc.winningBidsByBidder { + overallWinner := auc.winningBids[impId] + for bidderName, topBidPerBidder := range topBidsPerImp { + if overallWinner == topBidPerBidder { + topBidderLabel := metrics.AdapterLabels{ + Adapter: bidderName, + StoredImp: r.StoredImp, + } + + var cpm = float64(topBidPerBidder.bid.Price * 1000) + e.me.RecordAdapterWinningBidReceived(topBidderLabel, topBidPerBidder.bidType, topBidPerBidder.bid.AdM != "") + e.me.RecordAdapterWinningPrice(topBidderLabel, cpm) + } + + } + } + } + if !accountDebugAllow && !debugLog.DebugOverride { accountDebugDisabledWarning := openrtb_ext.ExtBidderMessage{ Code: errortypes.AccountLevelDebugDisabledWarningCode, @@ -960,10 +986,10 @@ func getPrimaryAdServer(adServerId int) (string, error) { // Extract all the data from the SeatBids and build the ExtBidResponse func (e *exchange) makeExtBidResponse(adapterBids map[openrtb_ext.BidderName]*pbsOrtbSeatBid, adapterExtra map[openrtb_ext.BidderName]*seatResponseExtra, r AuctionRequest, debugInfo bool, passthrough json.RawMessage, errList []error) *openrtb_ext.ExtBidResponse { bidResponseExt := &openrtb_ext.ExtBidResponse{ - Errors: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage, len(adapterBids)), - Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage, len(adapterBids)), - ResponseTimeMillis: make(map[openrtb_ext.BidderName]int, len(adapterBids)), - RequestTimeoutMillis: r.BidRequestWrapper.BidRequest.TMax, + Errors: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage, len(adapterBids)), + Warnings: make(map[openrtb_ext.BidderName][]openrtb_ext.ExtBidderMessage, len(adapterBids)), + ResponseTimeMillis: make(map[openrtb_ext.BidderName]int, len(adapterBids)), + // RequestTimeoutMillis: r.BidRequestWrapper.BidRequest.TMax, } if debugInfo { bidResponseExt.Debug = &openrtb_ext.ExtResponseDebug{ diff --git a/go.mod b/go.mod index be5ea481437..07521246007 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/IABTechLab/adscert v0.34.0 github.com/NYTimes/gziphandler v1.1.1 + github.com/ParticleMedia/ab-go-sdk v0.0.0-00010101000000-000000000000 github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/benbjohnson/clock v1.3.0 github.com/buger/jsonparser v1.1.1 @@ -24,50 +25,60 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/rs/cors v1.8.2 github.com/spf13/viper v1.12.0 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.0 github.com/vrischmann/go-metrics-influxdb v0.1.1 github.com/xeipuuv/gojsonschema v1.2.0 github.com/yudai/gojsondiff v1.0.0 - golang.org/x/net v0.0.0-20220909164309-bea034e7d591 - golang.org/x/text v0.3.7 + golang.org/x/net v0.1.0 + golang.org/x/text v0.4.0 google.golang.org/grpc v1.46.2 gopkg.in/evanphx/json-patch.v4 v4.12.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/antonmedv/expr v1.12.1 // indirect + github.com/aws/aws-sdk-go v1.44.217 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/mschoch/smat v0.2.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/sergi/go-diff v1.2.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/objx v0.4.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yudai/pp v2.0.1+incompatible // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect - golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/sys v0.1.0 // indirect google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) + +replace github.com/ParticleMedia/ab-go-sdk => ./ab-go-sdk diff --git a/go.sum b/go.sum index 5cce1e88dae..6fc5db18eb4 100644 --- a/go.sum +++ b/go.sum @@ -59,12 +59,16 @@ github.com/IABTechLab/adscert v0.34.0/go.mod h1:pCLd3Up1kfTrH6kYFUGGeavxIc1f6Tvv github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= +github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antonmedv/expr v1.12.1 h1:GTGrGN1kxxb+le0uQKaFRK8By4cvq1sleUCGE/U6hHg= +github.com/antonmedv/expr v1.12.1/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -73,6 +77,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.217 h1:FcWC56MRl+k756aH3qeMQTylSdeJ58WN0iFz3fkyRz0= +github.com/aws/aws-sdk-go v1.44.217/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -80,6 +86,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -287,7 +295,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -358,6 +368,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -422,6 +434,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= @@ -439,6 +453,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.8.2 h1:xehSyVa0YnHWsJ49JFljMpg1HX19V6NDZ1fkm1Xznbo= @@ -455,16 +471,18 @@ github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= @@ -488,6 +506,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= @@ -514,6 +533,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= @@ -554,6 +574,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -602,8 +623,9 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -632,6 +654,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -714,10 +737,13 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -727,8 +753,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -788,6 +815,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index a83266665f0..0219447520a 100644 --- a/main.go +++ b/main.go @@ -71,7 +71,8 @@ func serve(cfg *config.Configuration) error { currencyConverterTickerTask := task.NewTickerTask(fetchingInterval, currencyConverter) currencyConverterTickerTask.Start() - r, err := router.New(cfg, currencyConverter) + ab := task.NewAB(&http.Client{}) + r, err := router.New(cfg, currencyConverter, ab) if err != nil { return err } diff --git a/metrics/config/metrics.go b/metrics/config/metrics.go index ac49352d51d..1c7e828b9cd 100644 --- a/metrics/config/metrics.go +++ b/metrics/config/metrics.go @@ -156,6 +156,13 @@ func (me *MultiMetricsEngine) RecordAdapterBidReceived(labels metrics.AdapterLab } } +// RecordAdapterWinningBidReceived across all engines +func (me *MultiMetricsEngine) RecordAdapterWinningBidReceived(labels metrics.AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { + for _, thisME := range *me { + thisME.RecordAdapterWinningBidReceived(labels, bidType, hasAdm) + } +} + // RecordAdapterPrice across all engines func (me *MultiMetricsEngine) RecordAdapterPrice(labels metrics.AdapterLabels, cpm float64) { for _, thisME := range *me { @@ -163,6 +170,13 @@ func (me *MultiMetricsEngine) RecordAdapterPrice(labels metrics.AdapterLabels, c } } +// RecordAdapterWinningPrice across all engines +func (me *MultiMetricsEngine) RecordAdapterWinningPrice(labels metrics.AdapterLabels, cpm float64) { + for _, thisME := range *me { + thisME.RecordAdapterWinningPrice(labels, cpm) + } +} + // RecordAdapterTime across all engines func (me *MultiMetricsEngine) RecordAdapterTime(labels metrics.AdapterLabels, length time.Duration) { for _, thisME := range *me { @@ -335,10 +349,18 @@ func (me *NilMetricsEngine) RecordTLSHandshakeTime(tlsHandshakeTime time.Duratio func (me *NilMetricsEngine) RecordAdapterBidReceived(labels metrics.AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { } +// RecordAdapterWinningBidReceived as a noop +func (me *NilMetricsEngine) RecordAdapterWinningBidReceived(labels metrics.AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { +} + // RecordAdapterPrice as a noop func (me *NilMetricsEngine) RecordAdapterPrice(labels metrics.AdapterLabels, cpm float64) { } +// RecordAdapterWinningPrice as a noop +func (me *NilMetricsEngine) RecordAdapterWinningPrice(labels metrics.AdapterLabels, cpm float64) { +} + // RecordAdapterTime as a noop func (me *NilMetricsEngine) RecordAdapterTime(labels metrics.AdapterLabels, length time.Duration) { } diff --git a/metrics/go_metrics.go b/metrics/go_metrics.go index 9e6d8d1ba4f..270e9211e7c 100644 --- a/metrics/go_metrics.go +++ b/metrics/go_metrics.go @@ -79,19 +79,21 @@ type Metrics struct { // AdapterMetrics houses the metrics for a particular adapter type AdapterMetrics struct { - NoCookieMeter metrics.Meter - ErrorMeters map[AdapterError]metrics.Meter - NoBidMeter metrics.Meter - GotBidsMeter metrics.Meter - RequestTimer metrics.Timer - PriceHistogram metrics.Histogram - BidsReceivedMeter metrics.Meter - PanicMeter metrics.Meter - MarkupMetrics map[openrtb_ext.BidType]*MarkupDeliveryMetrics - ConnCreated metrics.Counter - ConnReused metrics.Counter - ConnWaitTime metrics.Timer - GDPRRequestBlocked metrics.Meter + NoCookieMeter metrics.Meter + ErrorMeters map[AdapterError]metrics.Meter + NoBidMeter metrics.Meter + GotBidsMeter metrics.Meter + RequestTimer metrics.Timer + PriceHistogram metrics.Histogram + WinningPriceHistogram metrics.Histogram + BidsReceivedMeter metrics.Meter + WinningBidsReceivedMeter metrics.Meter + PanicMeter metrics.Meter + MarkupMetrics map[openrtb_ext.BidType]*MarkupDeliveryMetrics + ConnCreated metrics.Counter + ConnReused metrics.Counter + ConnWaitTime metrics.Timer + GDPRRequestBlocked metrics.Meter } type MarkupDeliveryMetrics struct { @@ -318,15 +320,17 @@ func NewMetrics(registry metrics.Registry, exchanges []openrtb_ext.BidderName, d func makeBlankAdapterMetrics(disabledMetrics config.DisabledMetrics) *AdapterMetrics { blankMeter := &metrics.NilMeter{} newAdapter := &AdapterMetrics{ - NoCookieMeter: blankMeter, - ErrorMeters: make(map[AdapterError]metrics.Meter), - NoBidMeter: blankMeter, - GotBidsMeter: blankMeter, - RequestTimer: &metrics.NilTimer{}, - PriceHistogram: &metrics.NilHistogram{}, - BidsReceivedMeter: blankMeter, - PanicMeter: blankMeter, - MarkupMetrics: makeBlankBidMarkupMetrics(), + NoCookieMeter: blankMeter, + ErrorMeters: make(map[AdapterError]metrics.Meter), + NoBidMeter: blankMeter, + GotBidsMeter: blankMeter, + RequestTimer: &metrics.NilTimer{}, + PriceHistogram: &metrics.NilHistogram{}, + WinningPriceHistogram: &metrics.NilHistogram{}, + BidsReceivedMeter: blankMeter, + WinningBidsReceivedMeter: blankMeter, + PanicMeter: blankMeter, + MarkupMetrics: makeBlankBidMarkupMetrics(), } if !disabledMetrics.AdapterConnectionMetrics { newAdapter.ConnCreated = metrics.NilCounter{} @@ -364,6 +368,7 @@ func registerAdapterMetrics(registry metrics.Registry, adapterOrAccount string, am.GotBidsMeter = metrics.GetOrRegisterMeter(fmt.Sprintf("%[1]s.%[2]s.requests.gotbids", adapterOrAccount, exchange), registry) am.RequestTimer = metrics.GetOrRegisterTimer(fmt.Sprintf("%[1]s.%[2]s.request_time", adapterOrAccount, exchange), registry) am.PriceHistogram = metrics.GetOrRegisterHistogram(fmt.Sprintf("%[1]s.%[2]s.prices", adapterOrAccount, exchange), registry, metrics.NewExpDecaySample(1028, 0.015)) + am.WinningPriceHistogram = metrics.GetOrRegisterHistogram(fmt.Sprintf("%[1]s.%[2]s.prices", adapterOrAccount, exchange), registry, metrics.NewExpDecaySample(1028, 0.015)) am.MarkupMetrics = map[openrtb_ext.BidType]*MarkupDeliveryMetrics{ openrtb_ext.BidTypeBanner: makeDeliveryMetrics(registry, adapterOrAccount+"."+exchange, openrtb_ext.BidTypeBanner), openrtb_ext.BidTypeVideo: makeDeliveryMetrics(registry, adapterOrAccount+"."+exchange, openrtb_ext.BidTypeVideo), @@ -378,6 +383,7 @@ func registerAdapterMetrics(registry metrics.Registry, adapterOrAccount string, } if adapterOrAccount != "adapter" { am.BidsReceivedMeter = metrics.GetOrRegisterMeter(fmt.Sprintf("%[1]s.%[2]s.bids_received", adapterOrAccount, exchange), registry) + am.WinningBidsReceivedMeter = metrics.GetOrRegisterMeter(fmt.Sprintf("%[1]s.%[2]s.winning_bids_received", adapterOrAccount, exchange), registry) } am.PanicMeter = metrics.GetOrRegisterMeter(fmt.Sprintf("%[1]s.%[2]s.requests.panic", adapterOrAccount, exchange), registry) am.GDPRRequestBlocked = metrics.GetOrRegisterMeter(fmt.Sprintf("%[1]s.%[2]s.gdpr_request_blocked", adapterOrAccount, exchange), registry) @@ -624,6 +630,33 @@ func (me *Metrics) RecordAdapterBidReceived(labels AdapterLabels, bidType openrt } } +// RecordAdapterWinningBidReceived implements a part of the MetricsEngine interface. +// This tracks how many winning bids from each Bidder use `adm` vs. `nurl. +func (me *Metrics) RecordAdapterWinningBidReceived(labels AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { + am, ok := me.AdapterMetrics[labels.Adapter] + if !ok { + glog.Errorf("Trying to run adapter bid metrics on %s: adapter metrics not found", string(labels.Adapter)) + return + } + + // Adapter metrics + am.WinningBidsReceivedMeter.Mark(1) + // Account-Adapter metrics + if aam, ok := me.getAccountMetrics(labels.PubID).adapterMetrics[labels.Adapter]; ok { + aam.WinningBidsReceivedMeter.Mark(1) + } + + if metricsForType, ok := am.MarkupMetrics[bidType]; ok { + if hasAdm { + metricsForType.AdmMeter.Mark(1) + } else { + metricsForType.NurlMeter.Mark(1) + } + } else { + glog.Errorf("bid/adm metrics map entry does not exist for type %s. This is a bug, and should be reported.", bidType) + } +} + // RecordAdapterPrice implements a part of the MetricsEngine interface. Generates a histogram of winning bid prices func (me *Metrics) RecordAdapterPrice(labels AdapterLabels, cpm float64) { am, ok := me.AdapterMetrics[labels.Adapter] @@ -639,6 +672,21 @@ func (me *Metrics) RecordAdapterPrice(labels AdapterLabels, cpm float64) { } } +// RecordWinningAdapterPrice implements a part of the MetricsEngine interface. Generates a histogram of winning bid prices +func (me *Metrics) RecordAdapterWinningPrice(labels AdapterLabels, cpm float64) { + am, ok := me.AdapterMetrics[labels.Adapter] + if !ok { + glog.Errorf("Trying to run adapter price metrics on %s: adapter metrics not found", string(labels.Adapter)) + return + } + // Adapter metrics + am.WinningPriceHistogram.Update(int64(cpm)) + // Account-Adapter metrics + if aam, ok := me.getAccountMetrics(labels.PubID).adapterMetrics[labels.Adapter]; ok { + aam.WinningPriceHistogram.Update(int64(cpm)) + } +} + // RecordAdapterTime implements a part of the MetricsEngine interface. Records the adapter response time func (me *Metrics) RecordAdapterTime(labels AdapterLabels, length time.Duration) { am, ok := me.AdapterMetrics[labels.Adapter] diff --git a/metrics/metrics.go b/metrics/metrics.go index 42bec72fb52..83d2130232c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -13,6 +13,8 @@ type Labels struct { PubID string // exchange specific ID, so we cannot compile in values CookieFlag CookieFlag RequestStatus RequestStatus + StoredImp string + AbBuckets []string } // AdapterLabels defines the labels that can be attached to the adapter metrics. @@ -24,6 +26,8 @@ type AdapterLabels struct { CookieFlag CookieFlag AdapterBids AdapterBid AdapterErrors map[AdapterError]struct{} + StoredImp string + AbBuckets []string } // ImpLabels defines metric labels describing the impression type. @@ -134,6 +138,13 @@ type CacheResult string // PublisherUnknown : Default value for Labels.PubID const PublisherUnknown = "unknown" +// StoredImpUnknown: Defalut value for Labels.StoredImp +const StoredImpUnknown = "unknown" + +const AbBucketUnknown = "unknown" + +const LogUnknown = "unknown" + // The demand sources const ( DemandWeb DemandSource = "web" @@ -409,7 +420,9 @@ type MetricsEngine interface { // This records whether or not a bid of a particular type uses `adm` or `nurl`. // Since the legacy endpoints don't have a bid type, it can only count bids from OpenRTB and AMP. RecordAdapterBidReceived(labels AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) + RecordAdapterWinningBidReceived(labels AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) RecordAdapterPrice(labels AdapterLabels, cpm float64) + RecordAdapterWinningPrice(labels AdapterLabels, cpm float64) RecordAdapterTime(labels AdapterLabels, length time.Duration) RecordCookieSync(status CookieSyncStatus) RecordSyncerRequest(key string, status SyncerCookieSyncStatus) diff --git a/metrics/metrics_mock.go b/metrics/metrics_mock.go index 586a4783ef8..c551e8d20b9 100644 --- a/metrics/metrics_mock.go +++ b/metrics/metrics_mock.go @@ -76,11 +76,21 @@ func (me *MetricsEngineMock) RecordAdapterBidReceived(labels AdapterLabels, bidT me.Called(labels, bidType, hasAdm) } +// RecordAdapterWinningBidReceived mock +func (me *MetricsEngineMock) RecordAdapterWinningBidReceived(labels AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { + me.Called(labels, bidType, hasAdm) +} + // RecordAdapterPrice mock func (me *MetricsEngineMock) RecordAdapterPrice(labels AdapterLabels, cpm float64) { me.Called(labels, cpm) } +// RecordAdapterWinningPrice mock +func (me *MetricsEngineMock) RecordAdapterWinningPrice(labels AdapterLabels, cpm float64) { + me.Called(labels, cpm) +} + // RecordAdapterTime mock func (me *MetricsEngineMock) RecordAdapterTime(labels AdapterLabels, length time.Duration) { me.Called(labels, length) diff --git a/metrics/prometheus/preload.go b/metrics/prometheus/preload.go index bef88f20433..990d49612ab 100644 --- a/metrics/prometheus/preload.go +++ b/metrics/prometheus/preload.go @@ -23,6 +23,8 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { syncerRequestStatusValues = syncerRequestStatusesAsString() syncerSetsStatusValues = syncerSetStatusesAsString() sourceValues = []string{sourceRequest} + storedImpValues = []string{metrics.StoredImpUnknown} + abBucketValues = []string{metrics.AbBucketUnknown} ) preloadLabelValuesForCounter(m.connectionsError, map[string][]string{ @@ -51,10 +53,14 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { preloadLabelValuesForCounter(m.requests, map[string][]string{ requestTypeLabel: requestTypeValues, requestStatusLabel: requestStatusValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, }) preloadLabelValuesForHistogram(m.requestsTimer, map[string][]string{ requestTypeLabel: requestTypeValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, }) preloadLabelValuesForHistogram(m.storedAccountFetchTimer, map[string][]string{ @@ -124,6 +130,15 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { preloadLabelValuesForCounter(m.adapterBids, map[string][]string{ adapterLabel: adapterValues, markupDeliveryLabel: bidTypeValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, + }) + + preloadLabelValuesForCounter(m.adapterWinningBids, map[string][]string{ + adapterLabel: adapterValues, + markupDeliveryLabel: bidTypeValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, }) preloadLabelValuesForCounter(m.adapterErrors, map[string][]string{ @@ -136,13 +151,23 @@ func preloadLabelValues(m *Metrics, syncerKeys []string) { }) preloadLabelValuesForHistogram(m.adapterPrices, map[string][]string{ - adapterLabel: adapterValues, + adapterLabel: adapterValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, + }) + + preloadLabelValuesForHistogram(m.adapterWinningPrices, map[string][]string{ + adapterLabel: adapterValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, }) preloadLabelValuesForCounter(m.adapterRequests, map[string][]string{ - adapterLabel: adapterValues, - cookieLabel: cookieValues, - hasBidsLabel: boolValues, + adapterLabel: adapterValues, + cookieLabel: cookieValues, + hasBidsLabel: boolValues, + storedImpLabel: storedImpValues, + abLabel: abBucketValues, }) preloadLabelValuesForCounter(m.adsCertRequests, map[string][]string{ diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index df54df92248..7cd4fe2d609 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -59,9 +59,11 @@ type Metrics struct { // Adapter Metrics adapterBids *prometheus.CounterVec + adapterWinningBids *prometheus.CounterVec adapterErrors *prometheus.CounterVec adapterPanics *prometheus.CounterVec adapterPrices *prometheus.HistogramVec + adapterWinningPrices *prometheus.HistogramVec adapterRequests *prometheus.CounterVec adapterRequestsTimer *prometheus.HistogramVec adapterReusedConnections *prometheus.CounterVec @@ -104,6 +106,8 @@ const ( successLabel = "success" syncerLabel = "syncer" versionLabel = "version" + storedImpLabel = "stored_imp" + abLabel = "ab_bucket" ) const ( @@ -188,7 +192,7 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet metrics.requests = newCounter(cfg, reg, "requests", "Count of total requests to Prebid Server labeled by type and status.", - []string{requestTypeLabel, requestStatusLabel}) + []string{requestTypeLabel, requestStatusLabel, storedImpLabel, abLabel}) metrics.debugRequests = newCounterWithoutLabels(cfg, reg, "debug_requests", @@ -197,7 +201,7 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet metrics.requestsTimer = newHistogramVec(cfg, reg, "request_time_seconds", "Seconds to resolve successful Prebid Server requests labeled by type.", - []string{requestTypeLabel}, + []string{requestTypeLabel, storedImpLabel, abLabel}, standardTimeBuckets) metrics.requestsWithoutCookie = newCounter(cfg, reg, @@ -335,7 +339,12 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet metrics.adapterBids = newCounter(cfg, reg, "adapter_bids", "Count of bids labeled by adapter and markup delivery type (adm or nurl).", - []string{adapterLabel, markupDeliveryLabel}) + []string{adapterLabel, markupDeliveryLabel, storedImpLabel, abLabel}) + + metrics.adapterWinningBids = newCounter(cfg, reg, + "adapter_winning_bids", + "Count of winning bids labeled by adapter and markup delivery type (adm or nurl).", + []string{adapterLabel, markupDeliveryLabel, storedImpLabel, abLabel}) metrics.adapterErrors = newCounter(cfg, reg, "adapter_errors", @@ -350,13 +359,19 @@ func NewMetrics(cfg config.PrometheusMetrics, disabledMetrics config.DisabledMet metrics.adapterPrices = newHistogramVec(cfg, reg, "adapter_prices", "Monetary value of the bids labeled by adapter.", - []string{adapterLabel}, + []string{adapterLabel, storedImpLabel, abLabel}, + priceBuckets) + + metrics.adapterWinningPrices = newHistogramVec(cfg, reg, + "adapter_winning_prices", + "Monetary value of the bids labeled by adapter.", + []string{adapterLabel, storedImpLabel, abLabel}, priceBuckets) metrics.adapterRequests = newCounter(cfg, reg, "adapter_requests", "Count of requests labeled by adapter, if has a cookie, and if it resulted in bids.", - []string{adapterLabel, cookieLabel, hasBidsLabel}) + []string{adapterLabel, cookieLabel, hasBidsLabel, storedImpLabel, abLabel}) if !metrics.metricsDisabled.AdapterConnectionMetrics { metrics.adapterCreatedConnections = newCounter(cfg, reg, @@ -512,10 +527,14 @@ func (m *Metrics) RecordConnectionClose(success bool) { } func (m *Metrics) RecordRequest(labels metrics.Labels) { - m.requests.With(prometheus.Labels{ - requestTypeLabel: string(labels.RType), - requestStatusLabel: string(labels.RequestStatus), - }).Inc() + for _, bucket := range labels.AbBuckets { + m.requests.With(prometheus.Labels{ + requestTypeLabel: string(labels.RType), + requestStatusLabel: string(labels.RequestStatus), + storedImpLabel: string(labels.StoredImp), + abLabel: bucket, + }).Inc() + } if labels.CookieFlag == metrics.CookieFlagNo { m.requestsWithoutCookie.With(prometheus.Labels{ @@ -561,9 +580,13 @@ func (m *Metrics) RecordImps(labels metrics.ImpLabels) { func (m *Metrics) RecordRequestTime(labels metrics.Labels, length time.Duration) { if labels.RequestStatus == metrics.RequestStatusOK { - m.requestsTimer.With(prometheus.Labels{ - requestTypeLabel: string(labels.RType), - }).Observe(length.Seconds()) + for _, bucket := range labels.AbBuckets { + m.requestsTimer.With(prometheus.Labels{ + requestTypeLabel: string(labels.RType), + storedImpLabel: string(labels.StoredImp), + abLabel: bucket, + }).Observe(length.Seconds()) + } } } @@ -626,11 +649,15 @@ func (m *Metrics) RecordStoredDataError(labels metrics.StoredDataLabels) { } func (m *Metrics) RecordAdapterRequest(labels metrics.AdapterLabels) { - m.adapterRequests.With(prometheus.Labels{ - adapterLabel: string(labels.Adapter), - cookieLabel: string(labels.CookieFlag), - hasBidsLabel: strconv.FormatBool(labels.AdapterBids == metrics.AdapterBidPresent), - }).Inc() + for _, bucket := range labels.AbBuckets { + m.adapterRequests.With(prometheus.Labels{ + adapterLabel: string(labels.Adapter), + cookieLabel: string(labels.CookieFlag), + storedImpLabel: labels.StoredImp, + hasBidsLabel: strconv.FormatBool(labels.AdapterBids == metrics.AdapterBidPresent), + abLabel: bucket, + }).Inc() + } for err := range labels.AdapterErrors { m.adapterErrors.With(prometheus.Labels{ @@ -682,16 +709,51 @@ func (m *Metrics) RecordAdapterBidReceived(labels metrics.AdapterLabels, bidType markupDelivery = markupDeliveryAdm } - m.adapterBids.With(prometheus.Labels{ - adapterLabel: string(labels.Adapter), - markupDeliveryLabel: markupDelivery, - }).Inc() + for _, bucket := range labels.AbBuckets { + m.adapterBids.With(prometheus.Labels{ + adapterLabel: string(labels.Adapter), + markupDeliveryLabel: markupDelivery, + storedImpLabel: labels.StoredImp, + abLabel: bucket, + }).Inc() + } +} + +func (m *Metrics) RecordAdapterWinningBidReceived(labels metrics.AdapterLabels, bidType openrtb_ext.BidType, hasAdm bool) { + markupDelivery := markupDeliveryNurl + if hasAdm { + markupDelivery = markupDeliveryAdm + } + + for _, bucket := range labels.AbBuckets { + m.adapterWinningBids.With(prometheus.Labels{ + adapterLabel: string(labels.Adapter), + markupDeliveryLabel: markupDelivery, + storedImpLabel: labels.StoredImp, + abLabel: bucket, + }).Inc() + } } func (m *Metrics) RecordAdapterPrice(labels metrics.AdapterLabels, cpm float64) { - m.adapterPrices.With(prometheus.Labels{ - adapterLabel: string(labels.Adapter), - }).Observe(cpm) + for _, bucket := range labels.AbBuckets { + m.adapterPrices.With(prometheus.Labels{ + adapterLabel: string(labels.Adapter), + storedImpLabel: labels.StoredImp, + abLabel: bucket, + }).Observe(cpm) + } + +} + +func (m *Metrics) RecordAdapterWinningPrice(labels metrics.AdapterLabels, cpm float64) { + for _, bucket := range labels.AbBuckets { + m.adapterWinningPrices.With(prometheus.Labels{ + adapterLabel: string(labels.Adapter), + storedImpLabel: labels.StoredImp, + abLabel: bucket, + }).Observe(cpm) + } } func (m *Metrics) RecordAdapterTime(labels metrics.AdapterLabels, length time.Duration) { diff --git a/metrics/prometheus/prometheus_test.go b/metrics/prometheus/prometheus_test.go index 84516b4d460..037577ae1f2 100644 --- a/metrics/prometheus/prometheus_test.go +++ b/metrics/prometheus/prometheus_test.go @@ -2,6 +2,7 @@ package prometheusmetrics import ( "fmt" + "reflect" "testing" "time" @@ -141,10 +142,14 @@ func TestRequestMetric(t *testing.T) { m := createMetricsForTesting() requestType := metrics.ReqTypeORTB2Web requestStatus := metrics.RequestStatusBlacklisted + storedImp := "testImpName" + abBucket := []string{"unknown"} m.RecordRequest(metrics.Labels{ RType: requestType, RequestStatus: requestStatus, + StoredImp: storedImp, + AbBuckets: abBucket, }) expectedCount := float64(1) @@ -153,6 +158,8 @@ func TestRequestMetric(t *testing.T) { prometheus.Labels{ requestTypeLabel: string(requestType), requestStatusLabel: string(requestStatus), + storedImpLabel: storedImp, + abLabel: "unknown", }) } @@ -405,10 +412,14 @@ func TestImpressionsMetric(t *testing.T) { func TestRequestTimeMetric(t *testing.T) { requestType := metrics.ReqTypeORTB2Web + storedImp := "testImpName" + bucketName := "unknown" performTest := func(m *Metrics, requestStatus metrics.RequestStatus, timeInMs float64) { m.RecordRequestTime(metrics.Labels{ RType: requestType, RequestStatus: requestStatus, + StoredImp: storedImp, + AbBuckets: []string{bucketName}, }, time.Duration(timeInMs)*time.Millisecond) } @@ -440,8 +451,11 @@ func TestRequestTimeMetric(t *testing.T) { m := createMetricsForTesting() test.testCase(m) - - result := getHistogramFromHistogramVec(m.requestsTimer, requestTypeLabel, string(requestType)) + result := getHistogramFromHistogramMap(m.requestsTimer, map[string]string{ + requestTypeLabel: string(requestType), + storedImpLabel: storedImp, + abLabel: bucketName, + }) assertHistogram(t, test.description, result, test.expectedCount, test.expectedSum) } } @@ -650,9 +664,13 @@ func TestRecordStoredDataError(t *testing.T) { func TestAdapterBidReceivedMetric(t *testing.T) { adapterName := "anyName" + storedImpName := "testImpName" + abBucketName := "unknown" performTest := func(m *Metrics, hasAdm bool) { labels := metrics.AdapterLabels{ - Adapter: openrtb_ext.BidderName(adapterName), + Adapter: openrtb_ext.BidderName(adapterName), + StoredImp: storedImpName, + AbBuckets: []string{abBucketName}, } bidType := openrtb_ext.BidTypeBanner m.RecordAdapterBidReceived(labels, bidType, hasAdm) @@ -692,12 +710,16 @@ func TestAdapterBidReceivedMetric(t *testing.T) { prometheus.Labels{ adapterLabel: adapterName, markupDeliveryLabel: markupDeliveryAdm, + storedImpLabel: storedImpName, + abLabel: abBucketName, }) assertCounterVecValue(t, test.description, "adapterBids[nurl]", m.adapterBids, test.expectedNurlCount, prometheus.Labels{ adapterLabel: adapterName, markupDeliveryLabel: markupDeliveryNurl, + storedImpLabel: storedImpName, + abLabel: abBucketName, }) } } @@ -705,10 +727,12 @@ func TestAdapterBidReceivedMetric(t *testing.T) { func TestRecordAdapterPriceMetric(t *testing.T) { m := createMetricsForTesting() adapterName := "anyName" + bucketName := "unknown" cpm := float64(42) m.RecordAdapterPrice(metrics.AdapterLabels{ - Adapter: openrtb_ext.BidderName(adapterName), + Adapter: openrtb_ext.BidderName(adapterName), + AbBuckets: []string{bucketName}, }, cpm) expectedCount := uint64(1) @@ -719,11 +743,13 @@ func TestRecordAdapterPriceMetric(t *testing.T) { func TestAdapterRequestMetrics(t *testing.T) { adapterName := "anyName" + bucketName := "unknown" performTest := func(m *Metrics, cookieFlag metrics.CookieFlag, adapterBids metrics.AdapterBid) { labels := metrics.AdapterLabels{ Adapter: openrtb_ext.BidderName(adapterName), CookieFlag: cookieFlag, AdapterBids: adapterBids, + AbBuckets: []string{bucketName}, } m.RecordAdapterRequest(labels) } @@ -1608,6 +1634,20 @@ func getHistogramFromHistogramVecByTwoKeys(histogram *prometheus.HistogramVec, l return result } +func getHistogramFromHistogramMap(histogram *prometheus.HistogramVec, labels map[string]string) dto.Histogram { + var result dto.Histogram + processMetrics(histogram, func(m dto.Metric) { + metricsMap := map[string]string{} + for _, label := range m.Label { + metricsMap[label.GetName()] = label.GetValue() + } + if reflect.DeepEqual(metricsMap, labels) { + result = *m.GetHistogram() + } + }) + return result +} + func processMetrics(collector prometheus.Collector, handler func(m dto.Metric)) { collectorChan := make(chan prometheus.Metric) go func() { diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index e4c68fe62b1..2e26cea42ae 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -196,6 +196,7 @@ const ( BidderPangle BidderName = "pangle" BidderPGAM BidderName = "pgam" BidderPubmatic BidderName = "pubmatic" + BidderApplovin BidderName = "applovin" BidderPubnative BidderName = "pubnative" BidderPulsepoint BidderName = "pulsepoint" BidderQuantumdex BidderName = "quantumdex" @@ -363,6 +364,7 @@ func CoreBidderNames() []BidderName { BidderPangle, BidderPGAM, BidderPubmatic, + BidderApplovin, BidderPubnative, BidderPulsepoint, BidderQuantumdex, diff --git a/openrtb_ext/imp_applovin.go b/openrtb_ext/imp_applovin.go new file mode 100644 index 00000000000..7ba4d929835 --- /dev/null +++ b/openrtb_ext/imp_applovin.go @@ -0,0 +1,5 @@ +package openrtb_ext + +type ExtImpApplovin struct { + TagID string `json:"tagid"` +} diff --git a/router/router.go b/router/router.go index fb36238bd24..2e38c2b1e12 100644 --- a/router/router.go +++ b/router/router.go @@ -30,6 +30,7 @@ import ( "github.com/prebid/prebid-server/server/ssl" storedRequestsConf "github.com/prebid/prebid-server/stored_requests/config" "github.com/prebid/prebid-server/usersync" + "github.com/prebid/prebid-server/util/task" "github.com/prebid/prebid-server/util/uuidutil" "github.com/prebid/prebid-server/version" @@ -110,7 +111,7 @@ type Router struct { Shutdown func() } -func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *Router, err error) { +func New(cfg *config.Configuration, rateConvertor *currency.RateConverter, ab *task.AB) (r *Router, err error) { const schemaDirectory = "./static/bidder-params" r = &Router{ @@ -205,17 +206,17 @@ func New(cfg *config.Configuration, rateConvertor *currency.RateConverter) (r *R theExchange := exchange.NewExchange(adapters, cacheClient, cfg, syncersByBidder, r.MetricsEngine, cfg.BidderInfos, gdprPermsBuilder, tcf2CfgBuilder, rateConvertor, categoriesFetcher, adsCertSigner) var uuidGenerator uuidutil.UUIDRandomGenerator - openrtbEndpoint, err := openrtb2.NewEndpoint(uuidGenerator, theExchange, paramsValidator, fetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, storedRespFetcher) + openrtbEndpoint, err := openrtb2.NewEndpoint(uuidGenerator, theExchange, paramsValidator, fetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, storedRespFetcher, ab) if err != nil { glog.Fatalf("Failed to create the openrtb2 endpoint handler. %v", err) } - ampEndpoint, err := openrtb2.NewAmpEndpoint(uuidGenerator, theExchange, paramsValidator, ampFetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, storedRespFetcher) + ampEndpoint, err := openrtb2.NewAmpEndpoint(uuidGenerator, theExchange, paramsValidator, ampFetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, storedRespFetcher, ab) if err != nil { glog.Fatalf("Failed to create the amp endpoint handler. %v", err) } - videoEndpoint, err := openrtb2.NewVideoEndpoint(uuidGenerator, theExchange, paramsValidator, fetcher, videoFetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, cacheClient) + videoEndpoint, err := openrtb2.NewVideoEndpoint(uuidGenerator, theExchange, paramsValidator, fetcher, videoFetcher, accounts, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBidders, cacheClient, ab) if err != nil { glog.Fatalf("Failed to create the video endpoint handler. %v", err) } diff --git a/static/bidder-info/applovin.yaml b/static/bidder-info/applovin.yaml new file mode 100644 index 00000000000..449a4076784 --- /dev/null +++ b/static/bidder-info/applovin.yaml @@ -0,0 +1,15 @@ +endpoint: "https://s2s.applovin.com/ad" +disabled: true +maintainer: + email: "support@applovin.com" +capabilities: + app: + mediaTypes: + - banner + - video + - native + site: + mediaTypes: + - banner + - video + - native diff --git a/static/bidder-params/applovin.json b/static/bidder-params/applovin.json new file mode 100644 index 00000000000..28bd7f88e5b --- /dev/null +++ b/static/bidder-params/applovin.json @@ -0,0 +1,14 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Applovin Adapter Params", + "description": "A schema which validates params accepted by the Applovin adapter", + "type": "object", + "properties": { + "tagid": { + "type": "string", + "description": "Applovin token" + } + }, + "required": ["tagid"] + } + \ No newline at end of file diff --git a/stored_requests/backends/db_fetcher/fetcher.go b/stored_requests/backends/db_fetcher/fetcher.go index 1ad64a3ca3f..f82278a9b0f 100644 --- a/stored_requests/backends/db_fetcher/fetcher.go +++ b/stored_requests/backends/db_fetcher/fetcher.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "encoding/json" + "github.com/lib/pq" "github.com/golang/glog" @@ -34,6 +35,10 @@ type dbFetcher struct { responseQueryMaker func(numIds int) (query string) } +func (f *dbFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (fetcher *dbFetcher) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (map[string]json.RawMessage, map[string]json.RawMessage, []error) { if len(requestIDs) < 1 && len(impIDs) < 1 { return nil, nil, nil diff --git a/stored_requests/backends/empty_fetcher/fetcher.go b/stored_requests/backends/empty_fetcher/fetcher.go index 715a27089b0..a98ee1c2aa0 100644 --- a/stored_requests/backends/empty_fetcher/fetcher.go +++ b/stored_requests/backends/empty_fetcher/fetcher.go @@ -39,3 +39,14 @@ func (fetcher EmptyFetcher) FetchAccount(ctx context.Context, accountID string) func (fetcher EmptyFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } + +func (fetcher EmptyFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + errs = make([]error, 0, len(bucketList)) + for _, id := range bucketList { + errs = append(errs, stored_requests.NotFoundError{ + ID: id, + DataType: "Abs", + }) + } + return +} diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index 56677e264d9..470075b7035 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -33,6 +33,12 @@ func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []str return storedRequests, storedImpressions, errs } +func (fetcher *eagerFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + bucketData = fetcher.FileSystem.Directories["stored_abs"].Files + errs = appendErrors("AB", bucketList, bucketData, nil) + return +} + func (fetcher *eagerFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return nil, nil } diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index ed71ded417d..cecbdecc456 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -96,6 +96,10 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri return } +func (f *HttpFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (fetcher *HttpFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return nil, nil } diff --git a/stored_requests/config/config.go b/stored_requests/config/config.go index f091126800a..72f2add8be7 100644 --- a/stored_requests/config/config.go +++ b/stored_requests/config/config.go @@ -188,6 +188,7 @@ func newCache(cfg *config.StoredRequests) stored_requests.Cache { Imps: &nil_cache.NilCache{}, Responses: &nil_cache.NilCache{}, Accounts: &nil_cache.NilCache{}, + Abs: &nil_cache.NilCache{}, } switch { case cfg.InMemoryCache.Type == "none": @@ -198,6 +199,7 @@ func newCache(cfg *config.StoredRequests) stored_requests.Cache { cache.Requests = memory.NewCache(cfg.InMemoryCache.RequestCacheSize, cfg.InMemoryCache.TTL, "Requests") cache.Imps = memory.NewCache(cfg.InMemoryCache.ImpCacheSize, cfg.InMemoryCache.TTL, "Imps") cache.Responses = memory.NewCache(cfg.InMemoryCache.RespCacheSize, cfg.InMemoryCache.TTL, "Responses") + cache.Abs = memory.NewCache(cfg.InMemoryCache.RespCacheSize, cfg.InMemoryCache.TTL, "Abs") } return cache } diff --git a/stored_requests/fetcher.go b/stored_requests/fetcher.go index b1b4d2cf565..107138628e2 100644 --- a/stored_requests/fetcher.go +++ b/stored_requests/fetcher.go @@ -24,6 +24,7 @@ type Fetcher interface { // The returned objects can only be read from. They may not be written to. FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) + FetchABs(ctx context.Context, bucketList []string) (data map[string]json.RawMessage, errs []error) } type AccountFetcher interface { @@ -68,6 +69,7 @@ type Cache struct { Imps CacheJSON Responses CacheJSON Accounts CacheJSON + Abs CacheJSON } type CacheJSON interface { // Get works much like Fetcher.FetchRequests, with a few exceptions: @@ -163,6 +165,22 @@ func WithCache(fetcher AllFetcher, cache Cache, metricsEngine metrics.MetricsEng } } +func (f *fetcherWithCache) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + bucketData = f.cache.Abs.Get(ctx, bucketList) + leftoverABs := findLeftovers(bucketList, bucketData) + + if len(leftoverABs) > 0 { + fetcherABData, fetcherErrs := f.fetcher.FetchABs(ctx, leftoverABs) + errs = fetcherErrs + + f.cache.Abs.Save(ctx, fetcherABData) + + bucketData = mergeData(bucketData, fetcherABData) + } + + return +} + func (f *fetcherWithCache) FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) { requestData = f.cache.Requests.Get(ctx, requestIDs) diff --git a/stored_requests/fetcher_test.go b/stored_requests/fetcher_test.go index 4c0ed47c85a..49af4d5702f 100644 --- a/stored_requests/fetcher_test.go +++ b/stored_requests/fetcher_test.go @@ -17,9 +17,10 @@ func setupFetcherWithCacheDeps() (*mockCache, *mockCache, *mockCache, *mockFetch reqCache := &mockCache{} impCache := &mockCache{} respCache := &mockCache{} + abCache := &mockCache{} metricsEngine := &metrics.MetricsEngineMock{} fetcher := &mockFetcher{} - afetcherWithCache := WithCache(fetcher, Cache{reqCache, impCache, respCache, &nil_cache.NilCache{}}, metricsEngine) + afetcherWithCache := WithCache(fetcher, Cache{reqCache, impCache, respCache, &nil_cache.NilCache{}, abCache}, metricsEngine) return reqCache, impCache, respCache, fetcher, afetcherWithCache, metricsEngine } @@ -226,7 +227,7 @@ func setupAccountFetcherWithCacheDeps() (*mockCache, *mockFetcher, AllFetcher, * accCache := &mockCache{} metricsEngine := &metrics.MetricsEngineMock{} fetcher := &mockFetcher{} - afetcherWithCache := WithCache(fetcher, Cache{&nil_cache.NilCache{}, &nil_cache.NilCache{}, &nil_cache.NilCache{}, accCache}, metricsEngine) + afetcherWithCache := WithCache(fetcher, Cache{&nil_cache.NilCache{}, &nil_cache.NilCache{}, &nil_cache.NilCache{}, accCache, &nil_cache.NilCache{}}, metricsEngine) return accCache, fetcher, afetcherWithCache, metricsEngine } @@ -348,6 +349,10 @@ func (f *mockFetcher) FetchResponses(ctx context.Context, ids []string) (data ma return args.Get(0).(map[string]json.RawMessage), args.Get(1).([]error) } +func (f *mockFetcher) FetchABs(ctx context.Context, bucketList []string) (map[string]json.RawMessage, []error) { + return nil, nil +} + func (a *mockFetcher) FetchAccount(ctx context.Context, accountID string) (json.RawMessage, []error) { args := a.Called(ctx, accountID) return args.Get(0).(json.RawMessage), args.Get(1).([]error) diff --git a/stored_requests/multifetcher.go b/stored_requests/multifetcher.go index 70b1d62b8c1..e784206c472 100644 --- a/stored_requests/multifetcher.go +++ b/stored_requests/multifetcher.go @@ -36,6 +36,10 @@ func (mf MultiFetcher) FetchRequests(ctx context.Context, requestIDs []string, i return } +func (mf MultiFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (mf MultiFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return nil, nil } diff --git a/stored_responses/stored_responses_test.go b/stored_responses/stored_responses_test.go index 231bd187038..4fb0c1057b2 100644 --- a/stored_responses/stored_responses_test.go +++ b/stored_responses/stored_responses_test.go @@ -653,6 +653,10 @@ func (cf *mockStoredBidResponseFetcher) FetchRequests(ctx context.Context, reque return nil, nil, nil } +func (cf *mockStoredBidResponseFetcher) FetchABs(ctx context.Context, bucketList []string) (bucketData map[string]json.RawMessage, errs []error) { + return nil, nil +} + func (cf *mockStoredBidResponseFetcher) FetchResponses(ctx context.Context, ids []string) (data map[string]json.RawMessage, errs []error) { return cf.data, nil } diff --git a/util/task/ab.go b/util/task/ab.go new file mode 100644 index 00000000000..c00d5bcf4dd --- /dev/null +++ b/util/task/ab.go @@ -0,0 +1,71 @@ +package task + +import ( + "net/http" + "strconv" + "strings" + "sync/atomic" + + "github.com/ParticleMedia/ab-go-sdk" + "github.com/prebid/prebid-server/util/timeutil" +) + +const ab_url = "http://ab-api.ha.nb.com:8220" + +type AbResponse struct { + Abkv map[string]string `json:"exp"` +} + +type AB struct { + httpClient httpClient + lastUpdated atomic.Value + time timeutil.Time +} + +func NewAB( + httpClient httpClient, +) *AB { + ab.Init(&ab.ABConfig{ + App: "msp-prebid-server", + Url: ab_url, + Layers: []string{"ad_android", "ad_ios"}, + EnableCohort: false, // if you don't use cohort feature, please set to false + }) + return &AB{ + httpClient: httpClient, + lastUpdated: atomic.Value{}, + time: &timeutil.RealTime{}, + } +} + +type httpClient interface { + Do(req *http.Request) (*http.Response, error) +} + +func (abUtl *AB) GetBucket(uid string, os string, cv string) (error, []string) { + result := []string{} + os = strings.ToLower(os) + cv = strings.Replace(cv, ".", "", -1) + if len(cv) < 6 { + cv = cv + strings.Repeat("0", 6-len(cv)) + } + uidInt64, _ := strconv.ParseUint(uid, 10, 64) + response := ab.AB( + ab.NewABContext( + uid, + ).WithUserid( + uint32(uidInt64), + ).WithConditions(map[string]interface{}{ + "platform": os, + "cv": cv, + }), + ) + + for k, v := range response.Config { + result = append(result, strings.Replace(k+"_"+v, ".", "_", -1)) + } + + // all bucket is only used for metrics now, it can also be set in the future + result = append(result, "all") + return nil, result +}