Skip to content

feat: remove interfaces from protobuf models #243

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat: remove interfaces from protobuf models #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d4fa431
feat: read HTTP cookies as a map string to string
kruskall Mar 21, 2024
24dbfa6
test: update tests to use new cookies type
kruskall Mar 21, 2024
b5e8b42
Merge branch 'main' into feat/cookies-string
kruskall Apr 4, 2024
2d12b9d
feat: replace keyvalue with keyvaluestring protobuf message
kruskall Apr 4, 2024
a8c25f7
test: update tests
kruskall Apr 4, 2024
14052e8
feat: accept non-string values for http cookies
kruskall Apr 4, 2024
b8edebc
Merge branch 'main' into feat/cookies-string
kruskall Apr 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions codec/fullevent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func fullEvent(t *testing.B) *modelpb.APMEvent {
Span: &modelpb.Span{
Message: &modelpb.Message{
Body: "body",
Headers: []*modelpb.HTTPHeader{
Headers: []*modelpb.Header{
{
Key: "foo",
Value: []string{"bar"},
Expand Down Expand Up @@ -150,7 +150,7 @@ func fullEvent(t *testing.B) *modelpb.APMEvent {
},
Message: &modelpb.Message{
Body: "body",
Headers: []*modelpb.HTTPHeader{
Headers: []*modelpb.Header{
{
Key: "foo",
Value: []string{"bar"},
Expand Down Expand Up @@ -356,7 +356,7 @@ func fullEvent(t *testing.B) *modelpb.APMEvent {
Request: &modelpb.HTTPRequest{
Headers: randomHTTPHeaders(t),
Env: randomKv(t),
Cookies: randomKv(t),
Cookies: randomHTTPCookies(t),
Id: "id",
Method: "method",
Referrer: "referrer",
Expand Down
22 changes: 14 additions & 8 deletions codec/utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,12 @@ import (
"testing"

"github.com/elastic/apm-data/model/modelpb"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/structpb"
)

func randomKv(t testing.TB) []*modelpb.KeyValue {
value, err := structpb.NewValue(t.Name() + ".value." + randString())
require.NoError(t, err)
func randomKv(t testing.TB) []*modelpb.KeyValueString {
value := t.Name() + ".value." + randString()

kv := []*modelpb.KeyValue{
kv := []*modelpb.KeyValueString{
{
Key: t.Name() + ".key." + randString(),
Value: value,
Expand All @@ -40,15 +37,24 @@ func randomKv(t testing.TB) []*modelpb.KeyValue {
return kv
}

func randomHTTPHeaders(t testing.TB) []*modelpb.HTTPHeader {
return []*modelpb.HTTPHeader{
func randomHTTPHeaders(t testing.TB) []*modelpb.Header {
return []*modelpb.Header{
{
Key: t.Name() + ".key." + randString(),
Value: []string{t.Name() + ".value." + randString()},
},
}
}

func randomHTTPCookies(t testing.TB) []*modelpb.HTTPCookies {
return []*modelpb.HTTPCookies{
{
Key: t.Name() + ".key." + randString(),
Value: t.Name() + ".value." + randString(),
},
}
}

func uintPtr(i uint32) *uint32 {
return &i
}
Expand Down
14 changes: 7 additions & 7 deletions input/elasticapm/internal/modeldecoder/modeldecodertest/populator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ import (
"time"

"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/types/known/structpb"

"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder/nullable"
"github.com/elastic/apm-data/model/modelpb"
Expand Down Expand Up @@ -163,6 +162,8 @@ func SetStructValues(in interface{}, values *Values, opts ...SetStructValuesOpti
switch v := f.Interface().(type) {
case map[string]interface{}:
elemVal = reflect.ValueOf(values.Str)
case map[string]string:
elemVal = reflect.ValueOf(values.Str)
case map[string]float64:
elemVal = reflect.ValueOf(values.Float)
case map[string]*modelpb.LabelValue:
Expand Down Expand Up @@ -282,13 +283,12 @@ func AssertStructValues(t *testing.T, i interface{}, isException func(string) bo
m[fmt.Sprintf("%s%v", values.Str, i)] = &modelpb.LabelValue{Value: values.Str}
}
newVal = m
case []*modelpb.KeyValue:
m := []*modelpb.KeyValue{}
case []*modelpb.KeyValueString:
m := []*modelpb.KeyValueString{}
for i := 0; i < values.N; i++ {
value, _ := structpb.NewValue(values.Str)
m = append(m, &modelpb.KeyValue{
m = append(m, &modelpb.KeyValueString{
Key: fmt.Sprintf("%s%v", values.Str, i),
Value: value,
Value: values.Str,
})
}
newVal = m
Expand Down Expand Up @@ -440,7 +440,7 @@ func iterateStruct(v reflect.Value, key string, fn func(f reflect.Value, fKey st
}

switch f.Interface().(type) {
case []*modelpb.KeyValue:
case []*modelpb.KeyValueString:
continue
}

Expand Down
19 changes: 17 additions & 2 deletions input/elasticapm/internal/modeldecoder/modeldecoderutil/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package modeldecoderutil

import (
"encoding/json"
"fmt"
"net/http"

"github.com/elastic/apm-data/model/modelpb"
Expand All @@ -45,11 +46,25 @@ func HTTPHeadersToStructPb(h http.Header) *structpb.Struct {
return nil
}

func HTTPHeadersToModelpb(h http.Header, out []*modelpb.HTTPHeader) []*modelpb.HTTPHeader {
func HTTPCookiesToModelpb(m map[string]any, out []*modelpb.HTTPCookies) []*modelpb.HTTPCookies {
if len(m) == 0 {
return nil
}
out = Reslice(out, len(m), modelpb.HTTPCookiesFromVTPool)
i := 0
for k, v := range m {
out[i].Key = k
out[i].Value = fmt.Sprint(v)
i++
}
return out
}

func HTTPHeadersToModelpb(h http.Header, out []*modelpb.Header) []*modelpb.Header {
if len(h) == 0 {
return nil
}
out = Reslice(out, len(h), modelpb.HTTPHeaderFromVTPool)
out = Reslice(out, len(h), modelpb.HeaderFromVTPool)
i := 0
for k, v := range h {
out[i].Key = k
Expand Down
14 changes: 7 additions & 7 deletions input/elasticapm/internal/modeldecoder/modeldecoderutil/keyvaluepb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,23 +19,23 @@ package modeldecoderutil

import (
"github.com/elastic/apm-data/model/modelpb"
"google.golang.org/protobuf/types/known/structpb"
)

func ToKv(m map[string]any, out []*modelpb.KeyValue) []*modelpb.KeyValue {
func ToKv(m map[string]any, out []*modelpb.KeyValueString) []*modelpb.KeyValueString {
m = normalizeMap(m)
if len(m) == 0 {
return nil
}

out = Reslice(out, len(m), modelpb.KeyValueFromVTPool)
out = Reslice(out, len(m), modelpb.KeyValueStringFromVTPool)

i := 0
for k, v := range m {
value, _ := structpb.NewValue(v)
out[i].Key = k
out[i].Value = value
i++
if s, ok := v.(string); ok {
out[i].Key = k
out[i].Value = s
i++
}
}

return out
Expand Down
23 changes: 8 additions & 15 deletions input/elasticapm/internal/modeldecoder/modeldecoderutil/structpb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,11 @@ package modeldecoderutil

import (
"encoding/json"
"fmt"

"google.golang.org/protobuf/types/known/structpb"
)

func ToStruct(m map[string]any) *structpb.Struct {
nm := normalizeMap(m)
if len(nm) == 0 {
return nil
}

if str, err := structpb.NewStruct(nm); err == nil {
return str
}
return nil
}

func ToValue(a any) *structpb.Value {
nv := normalizeValue(a)
if nv == nil {
Expand All @@ -48,10 +37,14 @@ func ToValue(a any) *structpb.Value {
}

func normalizeMap(m map[string]any) map[string]any {
if v := normalizeValue(m); v != nil {
return v.(map[string]any)
for k, v := range m {
switch v := v.(type) {
case string:
default:
m[k] = fmt.Sprint(v)
}
}
return nil
return m
}

func normalizeValue(v interface{}) interface{} {
Expand Down
6 changes: 3 additions & 3 deletions input/elasticapm/internal/modeldecoder/rumv3/error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ func TestDecodeMapToErrorModel(t *testing.T) {
input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}})
var out modelpb.APMEvent
mapToErrorModel(&input, &out)
assert.Empty(t, cmp.Diff([]*modelpb.HTTPHeader{
assert.Empty(t, cmp.Diff([]*modelpb.Header{
{
Key: "a",
Value: []string{"b"},
Expand All @@ -216,13 +216,13 @@ func TestDecodeMapToErrorModel(t *testing.T) {
Value: []string{"d", "e"},
},
}, out.Http.Request.Headers,
cmpopts.SortSlices(func(x, y *modelpb.HTTPHeader) bool {
cmpopts.SortSlices(func(x, y *modelpb.Header) bool {
return x.Key < y.Key
}),
protocmp.Transform(),
))

assert.Equal(t, []*modelpb.HTTPHeader{
assert.Equal(t, []*modelpb.Header{
{
Key: "f",
Value: []string{"g"},
Expand Down
6 changes: 3 additions & 3 deletions input/elasticapm/internal/modeldecoder/rumv3/transaction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}})
var out modelpb.APMEvent
mapToTransactionModel(&input, &out)
assert.Empty(t, cmp.Diff([]*modelpb.HTTPHeader{
assert.Empty(t, cmp.Diff([]*modelpb.Header{
{
Key: "a",
Value: []string{"b"},
Expand All @@ -367,12 +367,12 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
Value: []string{"d", "e"},
},
}, out.Http.Request.Headers,
cmpopts.SortSlices(func(x, y *modelpb.HTTPHeader) bool {
cmpopts.SortSlices(func(x, y *modelpb.Header) bool {
return x.Key < y.Key
}),
protocmp.Transform(),
))
assert.Equal(t, []*modelpb.HTTPHeader{
assert.Equal(t, []*modelpb.Header{
{
Key: "f",
Value: []string{"g"},
Expand Down
2 changes: 1 addition & 1 deletion input/elasticapm/internal/modeldecoder/v2/decoder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -884,7 +884,7 @@ func mapToRequestModel(from contextRequest, out *modelpb.HTTPRequest) {
out.Body = modeldecoderutil.ToValue(modeldecoderutil.NormalizeHTTPRequestBody(from.Body.Val))
}
if len(from.Cookies) > 0 {
out.Cookies = modeldecoderutil.ToKv(from.Cookies, out.Cookies)
out.Cookies = modeldecoderutil.HTTPCookiesToModelpb(from.Cookies, out.Cookies)
}
if from.Headers.IsSet() {
out.Headers = modeldecoderutil.HTTPHeadersToModelpb(from.Headers.Val, out.Headers)
Expand Down
6 changes: 3 additions & 3 deletions input/elasticapm/internal/modeldecoder/v2/error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func TestDecodeMapToErrorModel(t *testing.T) {
input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}})
var out modelpb.APMEvent
mapToErrorModel(&input, &out)
assert.Empty(t, cmp.Diff([]*modelpb.HTTPHeader{
assert.Empty(t, cmp.Diff([]*modelpb.Header{
{
Key: "a",
Value: []string{"b"},
Expand All @@ -194,12 +194,12 @@ func TestDecodeMapToErrorModel(t *testing.T) {
Value: []string{"d", "e"},
},
}, out.Http.Request.Headers,
cmpopts.SortSlices(func(x, y *modelpb.HTTPHeader) bool {
cmpopts.SortSlices(func(x, y *modelpb.Header) bool {
return x.Key < y.Key
}),
protocmp.Transform(),
))
assert.Equal(t, []*modelpb.HTTPHeader{
assert.Equal(t, []*modelpb.Header{
{
Key: "f",
Value: []string{"g"},
Expand Down
2 changes: 1 addition & 1 deletion input/elasticapm/internal/modeldecoder/v2/span_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ func TestDecodeMapToSpanModel(t *testing.T) {
input.Context.HTTP.Response.Headers.Set(http.Header{"a": []string{"b", "c"}})
var out modelpb.APMEvent
mapToSpanModel(&input, &out)
assert.Equal(t, []*modelpb.HTTPHeader{
assert.Equal(t, []*modelpb.Header{
{
Key: "a",
Value: []string{"b", "c"},
Expand Down
6 changes: 3 additions & 3 deletions input/elasticapm/internal/modeldecoder/v2/transaction_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
input.Context.Response.Headers.Set(http.Header{"f": []string{"g"}})
var out modelpb.APMEvent
mapToTransactionModel(&input, &out)
assert.Empty(t, cmp.Diff([]*modelpb.HTTPHeader{
assert.Empty(t, cmp.Diff([]*modelpb.Header{
{
Key: "a",
Value: []string{"b"},
Expand All @@ -340,12 +340,12 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
Value: []string{"d", "e"},
},
}, out.Http.Request.Headers,
cmpopts.SortSlices(func(x, y *modelpb.HTTPHeader) bool {
cmpopts.SortSlices(func(x, y *modelpb.Header) bool {
return x.Key < y.Key
}),
protocmp.Transform(),
))
assert.Empty(t, cmp.Diff([]*modelpb.HTTPHeader{
assert.Empty(t, cmp.Diff([]*modelpb.Header{
{
Key: "f",
Value: []string{"g"},
Expand Down
6 changes: 3 additions & 3 deletions model/modeljson/apmevent.pb.json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func fullEvent(t testing.TB) *modelpb.APMEvent {
Span: &modelpb.Span{
Message: &modelpb.Message{
Body: "body",
Headers: []*modelpb.HTTPHeader{
Headers: []*modelpb.Header{
{
Key: "foo",
Value: []string{"bar"},
Expand Down Expand Up @@ -171,7 +171,7 @@ func fullEvent(t testing.TB) *modelpb.APMEvent {
},
Message: &modelpb.Message{
Body: "body",
Headers: []*modelpb.HTTPHeader{
Headers: []*modelpb.Header{
{
Key: "foo",
Value: []string{"bar"},
Expand Down Expand Up @@ -377,7 +377,7 @@ func fullEvent(t testing.TB) *modelpb.APMEvent {
Request: &modelpb.HTTPRequest{
Headers: randomHTTPHeaders(t),
Env: randomKv(t),
Cookies: randomKv(t),
Cookies: randomHTTPCookies(t),
Id: "id",
Method: "method",
Referrer: "referrer",
Expand Down
2 changes: 1 addition & 1 deletion model/modeljson/context.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import "github.com/elastic/apm-data/model/modelpb"

// updateFields transforms in, returning a copy with sanitized keys,
// suitable for storing as "custom" in transaction and error documents.
func updateFields(in []*modelpb.KeyValue) {
func updateFields(in []*modelpb.KeyValueString) {
for _, kv := range in {
kv.Key = sanitizeLabelKey(kv.Key)
}
Expand Down
2 changes: 1 addition & 1 deletion model/modeljson/http.pb.json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
func TestHTTPToModelJSON(t *testing.T) {
headers := randomHTTPHeaders(t)
headers2 := randomHTTPHeaders(t)
cookies := randomKv(t)
cookies := randomHTTPCookies(t)
envs := randomKv(t)
tru := true

Expand Down
2 changes: 1 addition & 1 deletion model/modeljson/internal/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type HTTPRequest struct {
Method string `json:"method,omitempty"`
Referrer string `json:"referrer,omitempty"`
Env KeyValueSlice `json:"env,omitempty"` // Non-ECS field.
Cookies KeyValueSlice `json:"cookies,omitempty"` // Non-ECS field.
Cookies HTTPCookies `json:"cookies,omitempty"` // Non-ECS field.
}

type HTTPRequestBody struct {
Expand Down
Loading