From 6472b4cb5d53c584196cd9ea7d044bd34d6fb0fe Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 09:38:55 +0700 Subject: [PATCH 01/21] include ObservationDomainId in netflowv9 --- producer/proto/producer_nf.go | 1 + 1 file changed, 1 insertion(+) diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index d57925cc..5a3f699e 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -791,6 +791,7 @@ func ProcessMessageNetFlowV9Config(packet *netflow.NFv9Packet, samplingRateSys S } fmsg.SequenceNum = seqnum fmsg.SamplingRate = uint64(samplingRate) + fmsg.ObservationDomainId = obsDomainId } return flowMessageSet, nil } From 44cfde2f045990544c5ba5d3e5306b1f5246047a Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 09:39:46 +0700 Subject: [PATCH 02/21] ip list renderer --- producer/proto/render.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/producer/proto/render.go b/producer/proto/render.go index 2e9070f2..a25f701b 100644 --- a/producer/proto/render.go +++ b/producer/proto/render.go @@ -46,6 +46,7 @@ var ( "NextHop": IPRenderer, "BgpNextHop": IPRenderer, "MplsLabelIp": IPRenderer, + "MplsIp": IPRenderer, "Etype": EtypeRenderer, "Proto": ProtoRenderer, "SrcNet": NetworkRenderer, @@ -155,6 +156,12 @@ func RenderIP(addr []byte) string { func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} { if dataC, ok := data.([]byte); ok { return RenderIP(dataC) + } else if dataC, ok := data.([][]byte); ok { + var ipList []string + for _, ip := range dataC { + ipList = append(ipList, RenderIP(ip)) + } + return ipList } return NilRenderer(msg, fieldName, data) } From 6d32e0c9e39452fc8a30c7d21fd8beb510c564cd Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 13:18:43 +0700 Subject: [PATCH 03/21] allocate early --- producer/proto/render.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/producer/proto/render.go b/producer/proto/render.go index a25f701b..815eb50c 100644 --- a/producer/proto/render.go +++ b/producer/proto/render.go @@ -157,9 +157,9 @@ func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) i if dataC, ok := data.([]byte); ok { return RenderIP(dataC) } else if dataC, ok := data.([][]byte); ok { - var ipList []string - for _, ip := range dataC { - ipList = append(ipList, RenderIP(ip)) + ipList := make([]string, len(dataC)) + for i, ip := range dataC { + ipList[i] = RenderIP(ip) } return ipList } From a4f42be783589f8ba79f4fb4cbf86597a2710f7c Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 15:39:21 +0700 Subject: [PATCH 04/21] actually render is capable of rendering slice's element add default rendering for MplsIp is enough --- producer/proto/render.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/producer/proto/render.go b/producer/proto/render.go index 815eb50c..6a03bcbf 100644 --- a/producer/proto/render.go +++ b/producer/proto/render.go @@ -156,12 +156,6 @@ func RenderIP(addr []byte) string { func IPRenderer(msg *ProtoProducerMessage, fieldName string, data interface{}) interface{} { if dataC, ok := data.([]byte); ok { return RenderIP(dataC) - } else if dataC, ok := data.([][]byte); ok { - ipList := make([]string, len(dataC)) - for i, ip := range dataC { - ipList[i] = RenderIP(ip) - } - return ipList } return NilRenderer(msg, fieldName, data) } From 1c10c4278d68b88e9dcb020845aaf58aba90292c Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 16:56:21 +0700 Subject: [PATCH 05/21] protection against uint64 overflow - some flow might have startime/endtime larger than uptime --- producer/proto/producer_nf.go | 8 ++++---- producer/proto/producer_test.go | 24 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index d57925cc..79a8f87b 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -544,15 +544,15 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba if err := DecodeUNumber(v, &timeFirstSwitched); err != nil { return err } - timeDiff := (uptime - timeFirstSwitched) - flowMessage.TimeFlowStartNs = baseTimeNs - uint64(timeDiff)*1000000000 + timeDiff := (int64(uptime) - int64(timeFirstSwitched)) + flowMessage.TimeFlowStartNs = uint64(int64(baseTime)*1000-timeDiff) * 1000000 case netflow.NFV9_FIELD_LAST_SWITCHED: var timeLastSwitched uint32 if err := DecodeUNumber(v, &timeLastSwitched); err != nil { return err } - timeDiff := (uptime - timeLastSwitched) - flowMessage.TimeFlowEndNs = baseTimeNs - uint64(timeDiff)*1000000000 + timeDiff := (int64(uptime) - int64(timeLastSwitched)) + flowMessage.TimeFlowEndNs = uint64(int64(baseTime)*1000-timeDiff) * 1000000 } } else if version == 10 { switch df.Type { diff --git a/producer/proto/producer_test.go b/producer/proto/producer_test.go index dc4d56b8..e6de30a6 100644 --- a/producer/proto/producer_test.go +++ b/producer/proto/producer_test.go @@ -18,6 +18,16 @@ func TestProcessMessageNetFlow(t *testing.T) { Type: netflow.NFV9_FIELD_IPV4_SRC_ADDR, Value: []byte{10, 0, 0, 1}, }, + netflow.DataField{ + Type: netflow.NFV9_FIELD_FIRST_SWITCHED, + // 218432176 + Value: []byte{0x0d, 0x05, 0x02, 0xb0}, + }, + netflow.DataField{ + Type: netflow.NFV9_FIELD_LAST_SWITCHED, + // 218432192 + Value: []byte{0x0d, 0x05, 0x02, 0xc0}, + }, }, }, } @@ -28,11 +38,19 @@ func TestProcessMessageNetFlow(t *testing.T) { } pktnf9 := netflow.NFv9Packet{ - FlowSets: dfs, + SystemUptime: 218432000, + UnixSeconds: 1705732882, + FlowSets: dfs, } testsr := &SingleSamplingRateSystem{1} - _, err := ProcessMessageNetFlowV9Config(&pktnf9, testsr, nil) - assert.Nil(t, err) + msgs, err := ProcessMessageNetFlowV9Config(&pktnf9, testsr, nil) + if assert.Nil(t, err) && assert.Len(t, msgs, 1) { + msg, ok := msgs[0].(*ProtoProducerMessage) + if assert.True(t, ok) { + assert.Equal(t, msg.TimeFlowStartNs, uint64(1705732882176000000)) + assert.Equal(t, msg.TimeFlowEndNs, uint64(1705732882192000000)) + } + } pktipfix := netflow.IPFIXPacket{ FlowSets: dfs, From 41e41c6c3e487baea43613a4d7d8ac923346eee9 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 20:24:30 +0700 Subject: [PATCH 06/21] fix mpls label decoding --- producer/proto/producer_nf.go | 8 ++++++-- producer/proto/producer_test.go | 11 +++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index 79a8f87b..2a48c83b 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -518,7 +518,9 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba return err } if len(flowMessage.MplsLabel) < 2 { - flowMessage.MplsLabel = make([]uint32, 2) + tmpLabels := make([]uint32, 2) + copy(tmpLabels, flowMessage.MplsLabel) + flowMessage.MplsLabel = tmpLabels } flowMessage.MplsLabel[1] = uint32(mplsLabel >> 4) case netflow.IPFIX_FIELD_mplsLabelStackSection3: @@ -527,7 +529,9 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba return err } if len(flowMessage.MplsLabel) < 3 { - flowMessage.MplsLabel = make([]uint32, 3) + tmpLabels := make([]uint32, 3) + copy(tmpLabels, flowMessage.MplsLabel) + flowMessage.MplsLabel = tmpLabels } flowMessage.MplsLabel[2] = uint32(mplsLabel >> 4) case netflow.IPFIX_FIELD_mplsTopLabelIPv4Address: diff --git a/producer/proto/producer_test.go b/producer/proto/producer_test.go index e6de30a6..38b82628 100644 --- a/producer/proto/producer_test.go +++ b/producer/proto/producer_test.go @@ -28,6 +28,16 @@ func TestProcessMessageNetFlow(t *testing.T) { // 218432192 Value: []byte{0x0d, 0x05, 0x02, 0xc0}, }, + netflow.DataField{ + Type: netflow.NFV9_FIELD_MPLS_LABEL_1, + // 24041 + Value: []byte{0x05, 0xde, 0x94}, + }, + netflow.DataField{ + Type: netflow.NFV9_FIELD_MPLS_LABEL_2, + // 211992 + Value: []byte{0x33, 0xc1, 0x85}, + }, }, }, } @@ -49,6 +59,7 @@ func TestProcessMessageNetFlow(t *testing.T) { if assert.True(t, ok) { assert.Equal(t, msg.TimeFlowStartNs, uint64(1705732882176000000)) assert.Equal(t, msg.TimeFlowEndNs, uint64(1705732882192000000)) + assert.Equal(t, msg.MplsLabel, []uint32{24041, 211992}) } } From a002158b79b1b6ef8e8c5c8f4e60f96c72a2c59c Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 26 Jan 2024 20:29:40 +0700 Subject: [PATCH 07/21] update mpls label test case to cover 3 stacks --- producer/proto/producer_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/producer/proto/producer_test.go b/producer/proto/producer_test.go index 38b82628..9db883e8 100644 --- a/producer/proto/producer_test.go +++ b/producer/proto/producer_test.go @@ -38,6 +38,11 @@ func TestProcessMessageNetFlow(t *testing.T) { // 211992 Value: []byte{0x33, 0xc1, 0x85}, }, + netflow.DataField{ + Type: netflow.NFV9_FIELD_MPLS_LABEL_3, + // 48675 + Value: []byte{0x0b, 0xe2, 0x35}, + }, }, }, } @@ -59,7 +64,7 @@ func TestProcessMessageNetFlow(t *testing.T) { if assert.True(t, ok) { assert.Equal(t, msg.TimeFlowStartNs, uint64(1705732882176000000)) assert.Equal(t, msg.TimeFlowEndNs, uint64(1705732882192000000)) - assert.Equal(t, msg.MplsLabel, []uint32{24041, 211992}) + assert.Equal(t, msg.MplsLabel, []uint32{24041, 211992, 48675}) } } From 7436e07bfd6d96fbfc525c155827f2b9f66fe334 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Wed, 14 Feb 2024 19:04:00 +0700 Subject: [PATCH 08/21] state engine --- go.mod | 12 ++++ go.sum | 86 +++++++++++++++++++++++++ state/badger.go | 109 ++++++++++++++++++++++++++++++++ state/memory.go | 49 +++++++++++++++ state/redis.go | 162 ++++++++++++++++++++++++++++++++++++++++++++++++ state/state.go | 62 ++++++++++++++++++ 6 files changed, 480 insertions(+) create mode 100644 state/badger.go create mode 100644 state/memory.go create mode 100644 state/redis.go create mode 100644 state/state.go diff --git a/go.mod b/go.mod index 717abb42..c7590f68 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,11 @@ go 1.20 require ( github.com/Shopify/sarama v1.38.1 + github.com/dgraph-io/badger/v4 v4.2.0 github.com/libp2p/go-reuseport v0.4.0 github.com/oschwald/geoip2-golang v1.9.0 github.com/prometheus/client_golang v1.18.0 + github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 github.com/xdg-go/scram v1.1.2 @@ -18,10 +20,18 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -35,6 +45,7 @@ require ( github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/oschwald/maxminddb-golang v1.11.0 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect @@ -42,6 +53,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect + go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/go.sum b/go.sum index 3dbf7b40..3ef66ee1 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,30 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= @@ -16,8 +32,25 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -40,6 +73,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -55,6 +90,8 @@ github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZI github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -67,6 +104,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -84,31 +123,61 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -121,9 +190,25 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -134,3 +219,4 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/state/badger.go b/state/badger.go new file mode 100644 index 00000000..435a080d --- /dev/null +++ b/state/badger.go @@ -0,0 +1,109 @@ +package state + +import ( + "encoding/json" + "github.com/dgraph-io/badger/v4" + "net/url" +) + +type badgerState[K comparable, V any] struct { + memory memoryState[K, V] + urlParsed *url.URL + db *badger.DB +} + +func (b *badgerState[K, V]) init() error { + db, err := badger.Open(badger.DefaultOptions(b.urlParsed.Path)) + if err != nil { + return err + } + b.db = db + // pre-populate local memory copy from existing badger data + err = b.db.View(func(txn *badger.Txn) error { + opts := badger.DefaultIteratorOptions + it := txn.NewIterator(opts) + defer it.Close() + for it.Rewind(); it.Valid(); it.Next() { + item := it.Item() + kRaw := item.Key() + err := item.Value(func(vRaw []byte) error { + var k K + var v V + if kErr := json.Unmarshal(kRaw, &k); kErr != nil { + return kErr + } + if vErr := json.Unmarshal(vRaw, &v); vErr != nil { + return vErr + } + return b.memory.Add(k, v) + }) + if err != nil { + return err + } + } + return nil + }) + return err +} + +func (b *badgerState[K, V]) Close() error { + return b.db.Close() +} + +func (b *badgerState[K, V]) Get(key K) (V, error) { + return b.memory.Get(key) +} + +func (b *badgerState[K, V]) Add(key K, value V) error { + return b.db.Update(func(txn *badger.Txn) error { + k, err := json.Marshal(key) + if err != nil { + return err + } + v, err := json.Marshal(value) + if err != nil { + return err + } + if err = txn.Set(k, v); err != nil { + return err + } + if err = b.memory.Add(key, value); err != nil { + return err + } + return nil + }) +} + +func (b *badgerState[K, V]) Delete(key K) error { + return b.db.Update(func(txn *badger.Txn) error { + k, err := json.Marshal(key) + if err != nil { + return err + } + if err = txn.Delete(k); err != nil { + return err + } + if err = b.memory.Delete(key); err != nil { + return err + } + return nil + }) +} + +func (b *badgerState[K, V]) Pop(key K) (V, error) { + var v V + err := b.db.Update(func(txn *badger.Txn) error { + k, err := json.Marshal(key) + if err != nil { + return err + } + if err = txn.Delete(k); err != nil { + return err + } + if v, err = b.memory.Pop(key); err != nil { + return err + } + return nil + }) + return v, err +} diff --git a/state/memory.go b/state/memory.go new file mode 100644 index 00000000..21ed5e56 --- /dev/null +++ b/state/memory.go @@ -0,0 +1,49 @@ +package state + +import ( + "sync" +) + +type memoryState[K comparable, V any] struct { + data map[K]V + lock *sync.RWMutex +} + +func (m *memoryState[K, V]) Close() error { + return nil +} + +func (m *memoryState[K, V]) Get(key K) (V, error) { + m.lock.RLock() + defer m.lock.RUnlock() + if v, ok := m.data[key]; ok { + return v, nil + } else { + return v, ErrorKeyNotFound + } +} + +func (m *memoryState[K, V]) Add(key K, value V) error { + m.lock.Lock() + defer m.lock.Unlock() + m.data[key] = value + return nil +} + +func (m *memoryState[K, V]) Delete(key K) error { + m.lock.Lock() + defer m.lock.Unlock() + delete(m.data, key) + return nil +} + +func (m *memoryState[K, V]) Pop(key K) (V, error) { + m.lock.Lock() + defer m.lock.Unlock() + if v, ok := m.data[key]; ok { + delete(m.data, key) + return v, nil + } else { + return v, ErrorKeyNotFound + } +} diff --git a/state/redis.go b/state/redis.go new file mode 100644 index 00000000..21fb1b32 --- /dev/null +++ b/state/redis.go @@ -0,0 +1,162 @@ +package state + +import ( + "context" + "encoding/json" + "fmt" + "github.com/redis/go-redis/v9" + "net/url" + "strconv" + "strings" + "sync" +) + +type redisState[K comparable, V any] struct { + memory memoryState[K, V] + urlParsed *url.URL + rPrefix string + db *redis.Client + ctx context.Context + cancel context.CancelFunc + wg *sync.WaitGroup +} + +const ( + redisOpAdd = 1 + redisOpDel = 2 +) + +func (r *redisState[K, V]) init() error { + r.rPrefix = r.urlParsed.Query().Get("prefix") + if r.rPrefix == "" { + return fmt.Errorf("'prefix' name is required on redis state engine, place it on your URL query string") + } + opts, err := redis.ParseURL(r.urlParsed.String()) + if err != nil { + return err + } + r.db = redis.NewClient(opts) + // pre-populate local memory copy from existing redis data + iter := r.db.Scan(r.ctx, 0, "*", 0).Iterator() + for iter.Next(r.ctx) { + kRaw := iter.Val() + res := r.db.Get(r.ctx, kRaw) + vRaw, err := res.Bytes() + if err != nil { + return err + } + var k K + var v V + kRaw, _ = strings.CutPrefix(kRaw, r.rPrefix) + if err = json.Unmarshal([]byte(kRaw), &k); err != nil { + return err + } + if err = json.Unmarshal(vRaw, &v); err != nil { + return err + } + if err = r.memory.Add(k, v); err != nil { + return err + } + } + if err = iter.Err(); err != nil { + return err + } + // subscribe to value changes + r.wg.Add(1) + go func() { + defer r.wg.Done() + ps := r.db.PSubscribe(r.ctx, fmt.Sprintf("%s*", r.rPrefix)) + defer ps.Close() + ch := ps.Channel() + mainLoop: + for { + select { + case msgRaw := <-ch: + op, err := strconv.Atoi(msgRaw.Payload) + if err != nil { + continue + } + keyRaw, _ := strings.CutPrefix(msgRaw.Channel, r.rPrefix) + var k K + if err = json.Unmarshal([]byte(keyRaw), &k); err != nil { + continue + } + switch op { + case redisOpAdd: + cmd := r.db.Get(r.ctx, msgRaw.Channel) + vBytes, err := cmd.Bytes() + if err != nil { + continue + } + var v V + if err = json.Unmarshal(vBytes, &v); err != nil { + continue + } + _ = r.memory.Add(k, v) + case redisOpDel: + _ = r.memory.Delete(k) + } + case <-r.ctx.Done(): + break mainLoop + } + } + }() + return nil +} + +func (r *redisState[K, V]) Close() error { + r.cancel() + r.wg.Wait() + return r.db.Close() +} + +func (r *redisState[K, V]) Get(key K) (V, error) { + return r.memory.Get(key) +} + +func (r *redisState[K, V]) Add(key K, value V) error { + k, err := json.Marshal(key) + if err != nil { + return err + } + v, err := json.Marshal(value) + if err != nil { + return err + } + kStr := fmt.Sprintf("%s%s", r.rPrefix, string(k)) + setStatus := r.db.Set(r.ctx, kStr, v, 0) + if err = setStatus.Err(); err != nil { + return err + } + pubStatus := r.db.Publish(r.ctx, kStr, redisOpAdd) + if err = pubStatus.Err(); err != nil { + return err + } + return nil +} + +func (r *redisState[K, V]) Delete(key K) error { + k, err := json.Marshal(key) + if err != nil { + return err + } + kStr := fmt.Sprintf("%s%s", r.rPrefix, string(k)) + delStatus := r.db.Del(r.ctx, kStr) + if err = delStatus.Err(); err != nil { + return err + } + pubStatus := r.db.Publish(r.ctx, kStr, redisOpDel) + if err = pubStatus.Err(); err != nil { + return err + } + return nil +} + +func (r *redisState[K, V]) Pop(key K) (V, error) { + v, err := r.Get(key) + if err != nil { + return v, err + } + err = r.Delete(key) + return v, err +} diff --git a/state/state.go b/state/state.go new file mode 100644 index 00000000..6b52824c --- /dev/null +++ b/state/state.go @@ -0,0 +1,62 @@ +package state + +import ( + "context" + "fmt" + "net/url" + "sync" +) + +var ( + SupportedStates = []string{"memory", "badger", "redis"} + ErrorKeyNotFound = fmt.Errorf("key not found") +) + +type State[K comparable, V any] interface { + Close() error + Get(key K) (V, error) + Add(key K, value V) error + Delete(key K) error + Pop(key K) (V, error) +} + +func NewState[K comparable, V any](rawUrl string) (State[K, V], error) { + urlParsed, err := url.Parse(rawUrl) + if err != nil { + return nil, err + } + memory := memoryState[K, V]{ + data: make(map[K]V), + lock: new(sync.RWMutex), + } + switch urlParsed.Scheme { + case "memory": + return &memory, nil + case "badger": + bd := &badgerState[K, V]{ + memory: memory, + urlParsed: urlParsed, + } + if err = bd.init(); err != nil { + return nil, err + } else { + return bd, nil + } + case "redis": + ctx, cancel := context.WithCancel(context.Background()) + rd := &redisState[K, V]{ + memory: memory, + urlParsed: urlParsed, + ctx: ctx, + cancel: cancel, + wg: new(sync.WaitGroup), + } + if err = rd.init(); err != nil { + return nil, err + } else { + return rd, nil + } + default: + return nil, fmt.Errorf("unknown state name %s", urlParsed.Scheme) + } +} From dda293f63a4e159d7909cd306dd1adc2471266af Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 17:10:49 +0700 Subject: [PATCH 09/21] sampling rate system --- cmd/goflow2/main.go | 9 ++++- producer/proto/producer_nf.go | 2 +- producer/proto/proto.go | 6 +-- state/redis.go | 73 ++++++++++++++++++++++++++--------- state/sampling_rate.go | 61 +++++++++++++++++++++++++++++ state/state.go | 4 +- 6 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 state/sampling_rate.go diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go index 53f46eb1..8e58aa73 100644 --- a/cmd/goflow2/main.go +++ b/cmd/goflow2/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "github.com/netsampler/goflow2/v2/state" "io" "net" "net/http" @@ -121,8 +122,12 @@ func main() { log.Fatal(err) } } - - flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, protoproducer.CreateSamplingSystem) + err = state.InitSamplingRate() + if err != nil { + log.Fatal(err) + } + defer state.CloseSamplingRate() + flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, state.CreateSamplingSystem) if err != nil { log.Fatal(err) } diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index d57925cc..ca3103c4 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -28,7 +28,7 @@ type basicSamplingRateSystem struct { samplinglock *sync.RWMutex } -func CreateSamplingSystem() SamplingRateSystem { +func CreateSamplingSystem(key string) SamplingRateSystem { ts := &basicSamplingRateSystem{ sampling: make(map[basicSamplingRateKey]uint32), samplinglock: &sync.RWMutex{}, diff --git a/producer/proto/proto.go b/producer/proto/proto.go index 47e15971..535c4457 100644 --- a/producer/proto/proto.go +++ b/producer/proto/proto.go @@ -14,7 +14,7 @@ type ProtoProducer struct { cfgMapped *producerConfigMapped samplinglock *sync.RWMutex sampling map[string]SamplingRateSystem - samplingRateSystem func() SamplingRateSystem + samplingRateSystem func(key string) SamplingRateSystem } func (p *ProtoProducer) enrich(flowMessageSet []producer.ProducerMessage, cb func(msg *ProtoProducerMessage)) { @@ -33,7 +33,7 @@ func (p *ProtoProducer) getSamplingRateSystem(args *producer.ProduceArgs) Sampli sampling, ok := p.sampling[key] p.samplinglock.RUnlock() if !ok { - sampling = p.samplingRateSystem() + sampling = p.samplingRateSystem(key) p.samplinglock.Lock() p.sampling[key] = sampling p.samplinglock.Unlock() @@ -95,7 +95,7 @@ func (p *ProtoProducer) Commit(flowMessageSet []producer.ProducerMessage) { func (p *ProtoProducer) Close() {} -func CreateProtoProducer(cfg *ProducerConfig, samplingRateSystem func() SamplingRateSystem) (producer.ProducerInterface, error) { +func CreateProtoProducer(cfg *ProducerConfig, samplingRateSystem func(key string) SamplingRateSystem) (producer.ProducerInterface, error) { cfgMapped, err := mapConfig(cfg) return &ProtoProducer{ cfgMapped: cfgMapped, diff --git a/state/redis.go b/state/redis.go index 21fb1b32..d9a5b7d2 100644 --- a/state/redis.go +++ b/state/redis.go @@ -9,16 +9,18 @@ import ( "strconv" "strings" "sync" + "time" ) type redisState[K comparable, V any] struct { - memory memoryState[K, V] - urlParsed *url.URL - rPrefix string - db *redis.Client - ctx context.Context - cancel context.CancelFunc - wg *sync.WaitGroup + memory memoryState[K, V] + urlParsed *url.URL + rPrefix string + refreshInterval int + db *redis.Client + ctx context.Context + cancel context.CancelFunc + wg *sync.WaitGroup } const ( @@ -26,17 +28,7 @@ const ( redisOpDel = 2 ) -func (r *redisState[K, V]) init() error { - r.rPrefix = r.urlParsed.Query().Get("prefix") - if r.rPrefix == "" { - return fmt.Errorf("'prefix' name is required on redis state engine, place it on your URL query string") - } - opts, err := redis.ParseURL(r.urlParsed.String()) - if err != nil { - return err - } - r.db = redis.NewClient(opts) - // pre-populate local memory copy from existing redis data +func (r *redisState[K, V]) populate() error { iter := r.db.Scan(r.ctx, 0, "*", 0).Iterator() for iter.Next(r.ctx) { kRaw := iter.Val() @@ -58,9 +50,52 @@ func (r *redisState[K, V]) init() error { return err } } - if err = iter.Err(); err != nil { + if err := iter.Err(); err != nil { + return err + } + return nil +} + +func (r *redisState[K, V]) init() error { + r.rPrefix = r.urlParsed.Query().Get("prefix") + if r.rPrefix == "" { + return fmt.Errorf("'prefix' name is required on redis state engine, place it on your URL query string") + } + opts, err := redis.ParseURL(r.urlParsed.String()) + if err != nil { return err } + r.db = redis.NewClient(opts) + interval := r.urlParsed.Query().Get("interval") + if interval == "" { + interval = "900" + } + r.refreshInterval, err = strconv.Atoi(interval) + if err != nil { + return err + } + // pre-populate local memory copy from existing redis data + if err = r.populate(); err != nil { + return err + } + // refresh goroutine + if r.refreshInterval > 0 { + r.wg.Add(1) + go func() { + defer r.wg.Done() + refreshCh := time.After(time.Duration(r.refreshInterval) * time.Second) + mainLoop: + for { + select { + case <-refreshCh: + _ = r.populate() + refreshCh = time.After(time.Duration(r.refreshInterval) * time.Second) + case <-r.ctx.Done(): + break mainLoop + } + } + }() + } // subscribe to value changes r.wg.Add(1) go func() { diff --git a/state/sampling_rate.go b/state/sampling_rate.go new file mode 100644 index 00000000..51d55823 --- /dev/null +++ b/state/sampling_rate.go @@ -0,0 +1,61 @@ +package state + +import ( + "flag" + "fmt" + "github.com/netsampler/goflow2/v2/producer/proto" + "net/url" + "strings" +) + +var StateSampling = flag.String("state.sampling", "memory://", fmt.Sprintf("Define state sampling rate engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) +var samplingRateDB State[samplingRateKey, uint32] + +type samplingRateKey struct { + Key string `json:"key"` + Version uint16 `json:"ver"` + ObsDomainId uint32 `json:"obs"` +} + +type SamplingRateSystem struct { + key string +} + +func (s *SamplingRateSystem) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) { + return samplingRateDB.Get(samplingRateKey{ + Key: s.key, + Version: version, + ObsDomainId: obsDomainId, + }) +} + +func (s *SamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) { + _ = samplingRateDB.Add(samplingRateKey{ + Key: s.key, + Version: version, + ObsDomainId: obsDomainId, + }, samplingRate) +} + +func CreateSamplingSystem(key string) protoproducer.SamplingRateSystem { + ts := &SamplingRateSystem{ + key: key, + } + return ts +} + +func InitSamplingRate() error { + samplingUrl, err := url.Parse(*StateSampling) + if err != nil { + return err + } + if !samplingUrl.Query().Has("prefix") { + samplingUrl.Query().Set("prefix", "goflow2:sampling_rate:") + } + samplingRateDB, err = NewState[samplingRateKey, uint32](samplingUrl.String()) + return err +} + +func CloseSamplingRate() error { + return samplingRateDB.Close() +} diff --git a/state/state.go b/state/state.go index 6b52824c..9338131f 100644 --- a/state/state.go +++ b/state/state.go @@ -8,7 +8,7 @@ import ( ) var ( - SupportedStates = []string{"memory", "badger", "redis"} + SupportedSchemes = []string{"memory", "badger", "redis"} ErrorKeyNotFound = fmt.Errorf("key not found") ) @@ -42,7 +42,7 @@ func NewState[K comparable, V any](rawUrl string) (State[K, V], error) { } else { return bd, nil } - case "redis": + case "redis", "rediss": ctx, cancel := context.WithCancel(context.Background()) rd := &redisState[K, V]{ memory: memory, From 2a5832b686e0305c53606649ce1c6fba6ca2fce9 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 18:39:36 +0700 Subject: [PATCH 10/21] template system --- cmd/goflow2/main.go | 16 ++-- metrics/templates.go | 3 +- state/templates.go | 160 +++++++++++++++++++++++++++++++++++ utils/templates/templates.go | 3 +- 4 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 state/templates.go diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go index 8e58aa73..b65b853e 100644 --- a/cmd/goflow2/main.go +++ b/cmd/goflow2/main.go @@ -122,11 +122,6 @@ func main() { log.Fatal(err) } } - err = state.InitSamplingRate() - if err != nil { - log.Fatal(err) - } - defer state.CloseSamplingRate() flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, state.CreateSamplingSystem) if err != nil { log.Fatal(err) @@ -183,6 +178,17 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) + err = state.InitSamplingRate() + if err != nil { + log.Fatal(err) + } + defer state.CloseSamplingRate() + err = state.InitTemplates() + if err != nil { + log.Fatal(err) + } + defer state.CloseTemplates() + var receivers []*utils.UDPReceiver var pipes []utils.FlowPipe diff --git a/metrics/templates.go b/metrics/templates.go index f5e21993..bc3949b0 100644 --- a/metrics/templates.go +++ b/metrics/templates.go @@ -1,6 +1,7 @@ package metrics import ( + "github.com/netsampler/goflow2/v2/state" "strconv" "github.com/netsampler/goflow2/v2/decoders/netflow" @@ -15,7 +16,7 @@ type PromTemplateSystem struct { // A default Prometheus template generator function to be used by a pipe func NewDefaultPromTemplateSystem(key string) netflow.NetFlowTemplateSystem { - return NewPromTemplateSystem(key, netflow.CreateTemplateSystem()) + return NewPromTemplateSystem(key, state.CreateTemplateSystem(key)) } // Creates a Prometheus template system that wraps another template system. diff --git a/state/templates.go b/state/templates.go new file mode 100644 index 00000000..1f93e921 --- /dev/null +++ b/state/templates.go @@ -0,0 +1,160 @@ +package state + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "github.com/netsampler/goflow2/v2/decoders/netflow" + "net/url" + "reflect" + "strings" +) + +var StateTemplates = flag.String("state.netflow.templates", "memory://", fmt.Sprintf("Define state templates engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) +var templatesDB State[templatesKey, templatesValue] + +type templatesKey struct { + Key string `json:"key"` + Version uint16 `json:"ver"` + ObsDomainId uint32 `json:"obs"` + TemplateID uint16 `json:"tid"` +} + +const ( + templateTypeTemplateRecord = 1 + templateTypeIPFIXOptionsTemplateRecord = 2 + templateTypeNFv9OptionsTemplateRecord = 3 +) + +type templatesValue struct { + TemplateType int `json:"ttype"` + Data interface{} `json:"data"` +} + +type templatesValueUnmarshal struct { + TemplateType int `json:"ttype"` + Data json.RawMessage `json:"data"` +} + +func (t *templatesValue) UnmarshalJSON(bytes []byte) error { + var v templatesValueUnmarshal + err := json.Unmarshal(bytes, &v) + if err != nil { + return err + } + t.TemplateType = v.TemplateType + switch v.TemplateType { + case templateTypeTemplateRecord: + var data netflow.TemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + case templateTypeIPFIXOptionsTemplateRecord: + var data netflow.IPFIXOptionsTemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + case templateTypeNFv9OptionsTemplateRecord: + var data netflow.NFv9OptionsTemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + default: + return fmt.Errorf("unknown template type: %d", v.TemplateType) + } + return nil +} + +type TemplateSystem struct { + key string +} + +func (t *TemplateSystem) RemoveTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { + if v, err := templatesDB.Pop(templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + }); err != nil && errors.Is(err, ErrorKeyNotFound) { + return nil, netflow.ErrorTemplateNotFound + } else if err != nil { + return nil, err + } else { + return v.Data, nil + } +} + +func (t *TemplateSystem) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { + if v, err := templatesDB.Get(templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + }); err != nil && errors.Is(err, ErrorKeyNotFound) { + return nil, netflow.ErrorTemplateNotFound + } else if err != nil { + return nil, err + } else { + return v.Data, nil + } +} + +func (t *TemplateSystem) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error { + k := templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + } + var err error + switch templatec := template.(type) { + case netflow.TemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeTemplateRecord, + Data: templatec, + }) + case netflow.IPFIXOptionsTemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeIPFIXOptionsTemplateRecord, + Data: templatec, + }) + case netflow.NFv9OptionsTemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeNFv9OptionsTemplateRecord, + Data: templatec, + }) + default: + return fmt.Errorf("unknown template type: %s", reflect.TypeOf(template).String()) + } + return err +} + +func CreateTemplateSystem(key string) netflow.NetFlowTemplateSystem { + ts := &TemplateSystem{ + key: key, + } + return ts +} + +func InitTemplates() error { + templatesUrl, err := url.Parse(*StateTemplates) + if err != nil { + return err + } + if !templatesUrl.Query().Has("prefix") { + templatesUrl.Query().Set("prefix", "goflow2:nf_templates:") + } + templatesDB, err = NewState[templatesKey, templatesValue](templatesUrl.String()) + return err +} + +func CloseTemplates() error { + return templatesDB.Close() +} diff --git a/utils/templates/templates.go b/utils/templates/templates.go index 0f61c80a..8259818a 100644 --- a/utils/templates/templates.go +++ b/utils/templates/templates.go @@ -2,6 +2,7 @@ package templates import ( "github.com/netsampler/goflow2/v2/decoders/netflow" + "github.com/netsampler/goflow2/v2/state" ) // Function that Create Template Systems. @@ -10,5 +11,5 @@ type TemplateSystemGenerator func(key string) netflow.NetFlowTemplateSystem // Default template generator func DefaultTemplateGenerator(key string) netflow.NetFlowTemplateSystem { - return netflow.CreateTemplateSystem() + return state.CreateTemplateSystem(key) } From 55691cc2bf78d69b9bb45d67de737d8d7e3b0d5d Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 19:05:29 +0700 Subject: [PATCH 11/21] import nits --- cmd/goflow2/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go index b65b853e..41421a4c 100644 --- a/cmd/goflow2/main.go +++ b/cmd/goflow2/main.go @@ -5,7 +5,6 @@ import ( "errors" "flag" "fmt" - "github.com/netsampler/goflow2/v2/state" "io" "net" "net/http" @@ -39,6 +38,7 @@ import ( // core libraries "github.com/netsampler/goflow2/v2/metrics" + "github.com/netsampler/goflow2/v2/state" "github.com/netsampler/goflow2/v2/utils" "github.com/netsampler/goflow2/v2/utils/debug" From bf6621a9dbf15a3af0db324d582a379ed018978d Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 19:21:06 +0700 Subject: [PATCH 12/21] remove badger as it has cli arg conflicts --- go.mod | 10 ---- go.sum | 80 ------------------------------ state/badger.go | 109 ----------------------------------------- state/sampling_rate.go | 3 +- state/state.go | 12 +---- state/templates.go | 3 +- 6 files changed, 5 insertions(+), 212 deletions(-) delete mode 100644 state/badger.go diff --git a/go.mod b/go.mod index c7590f68..79673e92 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.20 require ( github.com/Shopify/sarama v1.38.1 - github.com/dgraph-io/badger/v4 v4.2.0 github.com/libp2p/go-reuseport v0.4.0 github.com/oschwald/geoip2-golang v1.9.0 github.com/prometheus/client_golang v1.18.0 @@ -20,18 +19,11 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v1.12.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -45,7 +37,6 @@ require ( github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/oschwald/maxminddb-golang v1.11.0 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect @@ -53,7 +44,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - go.opencensus.io v0.22.5 // indirect golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.15.0 // indirect diff --git a/go.sum b/go.sum index 3ef66ee1..570a8fbd 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= @@ -7,24 +5,14 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= -github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= @@ -32,25 +20,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= -github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -73,8 +44,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -90,8 +59,6 @@ github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZI github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= @@ -123,61 +90,31 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -190,25 +127,9 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -219,4 +140,3 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/state/badger.go b/state/badger.go deleted file mode 100644 index 435a080d..00000000 --- a/state/badger.go +++ /dev/null @@ -1,109 +0,0 @@ -package state - -import ( - "encoding/json" - "github.com/dgraph-io/badger/v4" - "net/url" -) - -type badgerState[K comparable, V any] struct { - memory memoryState[K, V] - urlParsed *url.URL - db *badger.DB -} - -func (b *badgerState[K, V]) init() error { - db, err := badger.Open(badger.DefaultOptions(b.urlParsed.Path)) - if err != nil { - return err - } - b.db = db - // pre-populate local memory copy from existing badger data - err = b.db.View(func(txn *badger.Txn) error { - opts := badger.DefaultIteratorOptions - it := txn.NewIterator(opts) - defer it.Close() - for it.Rewind(); it.Valid(); it.Next() { - item := it.Item() - kRaw := item.Key() - err := item.Value(func(vRaw []byte) error { - var k K - var v V - if kErr := json.Unmarshal(kRaw, &k); kErr != nil { - return kErr - } - if vErr := json.Unmarshal(vRaw, &v); vErr != nil { - return vErr - } - return b.memory.Add(k, v) - }) - if err != nil { - return err - } - } - return nil - }) - return err -} - -func (b *badgerState[K, V]) Close() error { - return b.db.Close() -} - -func (b *badgerState[K, V]) Get(key K) (V, error) { - return b.memory.Get(key) -} - -func (b *badgerState[K, V]) Add(key K, value V) error { - return b.db.Update(func(txn *badger.Txn) error { - k, err := json.Marshal(key) - if err != nil { - return err - } - v, err := json.Marshal(value) - if err != nil { - return err - } - if err = txn.Set(k, v); err != nil { - return err - } - if err = b.memory.Add(key, value); err != nil { - return err - } - return nil - }) -} - -func (b *badgerState[K, V]) Delete(key K) error { - return b.db.Update(func(txn *badger.Txn) error { - k, err := json.Marshal(key) - if err != nil { - return err - } - if err = txn.Delete(k); err != nil { - return err - } - if err = b.memory.Delete(key); err != nil { - return err - } - return nil - }) -} - -func (b *badgerState[K, V]) Pop(key K) (V, error) { - var v V - err := b.db.Update(func(txn *badger.Txn) error { - k, err := json.Marshal(key) - if err != nil { - return err - } - if err = txn.Delete(k); err != nil { - return err - } - if v, err = b.memory.Pop(key); err != nil { - return err - } - return nil - }) - return v, err -} diff --git a/state/sampling_rate.go b/state/sampling_rate.go index 51d55823..7b43626d 100644 --- a/state/sampling_rate.go +++ b/state/sampling_rate.go @@ -3,9 +3,10 @@ package state import ( "flag" "fmt" - "github.com/netsampler/goflow2/v2/producer/proto" "net/url" "strings" + + "github.com/netsampler/goflow2/v2/producer/proto" ) var StateSampling = flag.String("state.sampling", "memory://", fmt.Sprintf("Define state sampling rate engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) diff --git a/state/state.go b/state/state.go index 9338131f..60e59dc0 100644 --- a/state/state.go +++ b/state/state.go @@ -8,7 +8,7 @@ import ( ) var ( - SupportedSchemes = []string{"memory", "badger", "redis"} + SupportedSchemes = []string{"memory", "redis"} ErrorKeyNotFound = fmt.Errorf("key not found") ) @@ -32,16 +32,6 @@ func NewState[K comparable, V any](rawUrl string) (State[K, V], error) { switch urlParsed.Scheme { case "memory": return &memory, nil - case "badger": - bd := &badgerState[K, V]{ - memory: memory, - urlParsed: urlParsed, - } - if err = bd.init(); err != nil { - return nil, err - } else { - return bd, nil - } case "redis", "rediss": ctx, cancel := context.WithCancel(context.Background()) rd := &redisState[K, V]{ diff --git a/state/templates.go b/state/templates.go index 1f93e921..484fabaa 100644 --- a/state/templates.go +++ b/state/templates.go @@ -5,10 +5,11 @@ import ( "errors" "flag" "fmt" - "github.com/netsampler/goflow2/v2/decoders/netflow" "net/url" "reflect" "strings" + + "github.com/netsampler/goflow2/v2/decoders/netflow" ) var StateTemplates = flag.String("state.netflow.templates", "memory://", fmt.Sprintf("Define state templates engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) From 6c546df10656bf39a71bb3c51532bf22f17e35cc Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 19:45:01 +0700 Subject: [PATCH 13/21] Revert "Merge branch 'main' into iqbal/state" This reverts commit b30e24fae55b5f3b8ed6687be2b022cd3e99b1f9, reversing changes made to 2a5832b686e0305c53606649ce1c6fba6ca2fce9. --- producer/proto/producer_nf.go | 17 +++++--------- producer/proto/producer_test.go | 40 +++------------------------------ producer/proto/render.go | 1 - 3 files changed, 9 insertions(+), 49 deletions(-) diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index ed3cee3c..ca3103c4 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -518,9 +518,7 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba return err } if len(flowMessage.MplsLabel) < 2 { - tmpLabels := make([]uint32, 2) - copy(tmpLabels, flowMessage.MplsLabel) - flowMessage.MplsLabel = tmpLabels + flowMessage.MplsLabel = make([]uint32, 2) } flowMessage.MplsLabel[1] = uint32(mplsLabel >> 4) case netflow.IPFIX_FIELD_mplsLabelStackSection3: @@ -529,9 +527,7 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba return err } if len(flowMessage.MplsLabel) < 3 { - tmpLabels := make([]uint32, 3) - copy(tmpLabels, flowMessage.MplsLabel) - flowMessage.MplsLabel = tmpLabels + flowMessage.MplsLabel = make([]uint32, 3) } flowMessage.MplsLabel[2] = uint32(mplsLabel >> 4) case netflow.IPFIX_FIELD_mplsTopLabelIPv4Address: @@ -548,15 +544,15 @@ func ConvertNetFlowDataSet(flowMessage *ProtoProducerMessage, version uint16, ba if err := DecodeUNumber(v, &timeFirstSwitched); err != nil { return err } - timeDiff := (int64(uptime) - int64(timeFirstSwitched)) - flowMessage.TimeFlowStartNs = uint64(int64(baseTime)*1000-timeDiff) * 1000000 + timeDiff := (uptime - timeFirstSwitched) + flowMessage.TimeFlowStartNs = baseTimeNs - uint64(timeDiff)*1000000000 case netflow.NFV9_FIELD_LAST_SWITCHED: var timeLastSwitched uint32 if err := DecodeUNumber(v, &timeLastSwitched); err != nil { return err } - timeDiff := (int64(uptime) - int64(timeLastSwitched)) - flowMessage.TimeFlowEndNs = uint64(int64(baseTime)*1000-timeDiff) * 1000000 + timeDiff := (uptime - timeLastSwitched) + flowMessage.TimeFlowEndNs = baseTimeNs - uint64(timeDiff)*1000000000 } } else if version == 10 { switch df.Type { @@ -795,7 +791,6 @@ func ProcessMessageNetFlowV9Config(packet *netflow.NFv9Packet, samplingRateSys S } fmsg.SequenceNum = seqnum fmsg.SamplingRate = uint64(samplingRate) - fmsg.ObservationDomainId = obsDomainId } return flowMessageSet, nil } diff --git a/producer/proto/producer_test.go b/producer/proto/producer_test.go index 9db883e8..dc4d56b8 100644 --- a/producer/proto/producer_test.go +++ b/producer/proto/producer_test.go @@ -18,31 +18,6 @@ func TestProcessMessageNetFlow(t *testing.T) { Type: netflow.NFV9_FIELD_IPV4_SRC_ADDR, Value: []byte{10, 0, 0, 1}, }, - netflow.DataField{ - Type: netflow.NFV9_FIELD_FIRST_SWITCHED, - // 218432176 - Value: []byte{0x0d, 0x05, 0x02, 0xb0}, - }, - netflow.DataField{ - Type: netflow.NFV9_FIELD_LAST_SWITCHED, - // 218432192 - Value: []byte{0x0d, 0x05, 0x02, 0xc0}, - }, - netflow.DataField{ - Type: netflow.NFV9_FIELD_MPLS_LABEL_1, - // 24041 - Value: []byte{0x05, 0xde, 0x94}, - }, - netflow.DataField{ - Type: netflow.NFV9_FIELD_MPLS_LABEL_2, - // 211992 - Value: []byte{0x33, 0xc1, 0x85}, - }, - netflow.DataField{ - Type: netflow.NFV9_FIELD_MPLS_LABEL_3, - // 48675 - Value: []byte{0x0b, 0xe2, 0x35}, - }, }, }, } @@ -53,20 +28,11 @@ func TestProcessMessageNetFlow(t *testing.T) { } pktnf9 := netflow.NFv9Packet{ - SystemUptime: 218432000, - UnixSeconds: 1705732882, - FlowSets: dfs, + FlowSets: dfs, } testsr := &SingleSamplingRateSystem{1} - msgs, err := ProcessMessageNetFlowV9Config(&pktnf9, testsr, nil) - if assert.Nil(t, err) && assert.Len(t, msgs, 1) { - msg, ok := msgs[0].(*ProtoProducerMessage) - if assert.True(t, ok) { - assert.Equal(t, msg.TimeFlowStartNs, uint64(1705732882176000000)) - assert.Equal(t, msg.TimeFlowEndNs, uint64(1705732882192000000)) - assert.Equal(t, msg.MplsLabel, []uint32{24041, 211992, 48675}) - } - } + _, err := ProcessMessageNetFlowV9Config(&pktnf9, testsr, nil) + assert.Nil(t, err) pktipfix := netflow.IPFIXPacket{ FlowSets: dfs, diff --git a/producer/proto/render.go b/producer/proto/render.go index 6a03bcbf..2e9070f2 100644 --- a/producer/proto/render.go +++ b/producer/proto/render.go @@ -46,7 +46,6 @@ var ( "NextHop": IPRenderer, "BgpNextHop": IPRenderer, "MplsLabelIp": IPRenderer, - "MplsIp": IPRenderer, "Etype": EtypeRenderer, "Proto": ProtoRenderer, "SrcNet": NetworkRenderer, From ccdc7d0bab2410519a0b1d49649a51201110b6d5 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 19:49:12 +0700 Subject: [PATCH 14/21] import nits --- metrics/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/templates.go b/metrics/templates.go index bc3949b0..c35540c3 100644 --- a/metrics/templates.go +++ b/metrics/templates.go @@ -1,10 +1,10 @@ package metrics import ( - "github.com/netsampler/goflow2/v2/state" "strconv" "github.com/netsampler/goflow2/v2/decoders/netflow" + "github.com/netsampler/goflow2/v2/state" "github.com/prometheus/client_golang/prometheus" ) From 8d18a13abc5f4b986498a451b6c83c094ea616a8 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Thu, 15 Feb 2024 19:52:35 +0700 Subject: [PATCH 15/21] fix redis --- state/redis.go | 22 +++++++++++++++------- state/sampling_rate.go | 4 +++- state/templates.go | 4 +++- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/state/redis.go b/state/redis.go index d9a5b7d2..02837a0a 100644 --- a/state/redis.go +++ b/state/redis.go @@ -29,7 +29,7 @@ const ( ) func (r *redisState[K, V]) populate() error { - iter := r.db.Scan(r.ctx, 0, "*", 0).Iterator() + iter := r.db.Scan(r.ctx, 0, fmt.Sprintf("%s*", r.rPrefix), 0).Iterator() for iter.Next(r.ctx) { kRaw := iter.Val() res := r.db.Get(r.ctx, kRaw) @@ -57,20 +57,28 @@ func (r *redisState[K, V]) populate() error { } func (r *redisState[K, V]) init() error { - r.rPrefix = r.urlParsed.Query().Get("prefix") + q := r.urlParsed.Query() + r.rPrefix = q.Get("prefix") if r.rPrefix == "" { return fmt.Errorf("'prefix' name is required on redis state engine, place it on your URL query string") } + q.Del("prefix") + interval := q.Get("interval") + if interval == "" { + interval = "900" + } + q.Del("interval") + r.urlParsed.RawQuery = q.Encode() + var err error + r.refreshInterval, err = strconv.Atoi(interval) + if err != nil { + return err + } opts, err := redis.ParseURL(r.urlParsed.String()) if err != nil { return err } r.db = redis.NewClient(opts) - interval := r.urlParsed.Query().Get("interval") - if interval == "" { - interval = "900" - } - r.refreshInterval, err = strconv.Atoi(interval) if err != nil { return err } diff --git a/state/sampling_rate.go b/state/sampling_rate.go index 7b43626d..a13194f6 100644 --- a/state/sampling_rate.go +++ b/state/sampling_rate.go @@ -51,7 +51,9 @@ func InitSamplingRate() error { return err } if !samplingUrl.Query().Has("prefix") { - samplingUrl.Query().Set("prefix", "goflow2:sampling_rate:") + q := samplingUrl.Query() + q.Set("prefix", "goflow2:sampling_rate:") + samplingUrl.RawQuery = q.Encode() } samplingRateDB, err = NewState[samplingRateKey, uint32](samplingUrl.String()) return err diff --git a/state/templates.go b/state/templates.go index 484fabaa..75236fce 100644 --- a/state/templates.go +++ b/state/templates.go @@ -150,7 +150,9 @@ func InitTemplates() error { return err } if !templatesUrl.Query().Has("prefix") { - templatesUrl.Query().Set("prefix", "goflow2:nf_templates:") + q := templatesUrl.Query() + q.Set("prefix", "goflow2:nf_templates:") + templatesUrl.RawQuery = q.Encode() } templatesDB, err = NewState[templatesKey, templatesValue](templatesUrl.String()) return err From 03f0a2c7d0ffad4b16666605421d2d9f7415af7c Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 12:02:37 +0700 Subject: [PATCH 16/21] fix & benchmark --- state/sampling_rate.go | 16 +++++++++++++++- state/templates.go | 29 +++++++++++++++++++++++++++- state/templates_test.go | 42 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 state/templates_test.go diff --git a/state/sampling_rate.go b/state/sampling_rate.go index a13194f6..bf856d87 100644 --- a/state/sampling_rate.go +++ b/state/sampling_rate.go @@ -5,12 +5,14 @@ import ( "fmt" "net/url" "strings" + "sync" "github.com/netsampler/goflow2/v2/producer/proto" ) var StateSampling = flag.String("state.sampling", "memory://", fmt.Sprintf("Define state sampling rate engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) var samplingRateDB State[samplingRateKey, uint32] +var samplingRateInitLock = new(sync.Mutex) type samplingRateKey struct { Key string `json:"key"` @@ -46,6 +48,11 @@ func CreateSamplingSystem(key string) protoproducer.SamplingRateSystem { } func InitSamplingRate() error { + samplingRateInitLock.Lock() + defer samplingRateInitLock.Unlock() + if samplingRateDB != nil { + return nil + } samplingUrl, err := url.Parse(*StateSampling) if err != nil { return err @@ -60,5 +67,12 @@ func InitSamplingRate() error { } func CloseSamplingRate() error { - return samplingRateDB.Close() + samplingRateInitLock.Lock() + defer samplingRateInitLock.Unlock() + if samplingRateDB == nil { + return nil + } + err := samplingRateDB.Close() + samplingRateDB = nil + return err } diff --git a/state/templates.go b/state/templates.go index 75236fce..6c698f2e 100644 --- a/state/templates.go +++ b/state/templates.go @@ -8,12 +8,14 @@ import ( "net/url" "reflect" "strings" + "sync" "github.com/netsampler/goflow2/v2/decoders/netflow" ) var StateTemplates = flag.String("state.netflow.templates", "memory://", fmt.Sprintf("Define state templates engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) var templatesDB State[templatesKey, templatesValue] +var templatesInitLock = new(sync.Mutex) type templatesKey struct { Key string `json:"key"` @@ -23,6 +25,7 @@ type templatesKey struct { } const ( + templateTypeTest = 0 templateTypeTemplateRecord = 1 templateTypeIPFIXOptionsTemplateRecord = 2 templateTypeNFv9OptionsTemplateRecord = 3 @@ -46,6 +49,13 @@ func (t *templatesValue) UnmarshalJSON(bytes []byte) error { } t.TemplateType = v.TemplateType switch v.TemplateType { + case templateTypeTest: + var data int + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data case templateTypeTemplateRecord: var data netflow.TemplateRecord err = json.Unmarshal(v.Data, &data) @@ -131,6 +141,11 @@ func (t *TemplateSystem) AddTemplate(version uint16, obsDomainId uint32, templat TemplateType: templateTypeNFv9OptionsTemplateRecord, Data: templatec, }) + case int: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeTest, + Data: templatec, + }) default: return fmt.Errorf("unknown template type: %s", reflect.TypeOf(template).String()) } @@ -145,6 +160,11 @@ func CreateTemplateSystem(key string) netflow.NetFlowTemplateSystem { } func InitTemplates() error { + templatesInitLock.Lock() + defer templatesInitLock.Unlock() + if templatesDB != nil { + return nil + } templatesUrl, err := url.Parse(*StateTemplates) if err != nil { return err @@ -159,5 +179,12 @@ func InitTemplates() error { } func CloseTemplates() error { - return templatesDB.Close() + templatesInitLock.Lock() + defer templatesInitLock.Unlock() + if templatesDB == nil { + return nil + } + err := templatesDB.Close() + templatesDB = nil + return err } diff --git a/state/templates_test.go b/state/templates_test.go new file mode 100644 index 00000000..648690ee --- /dev/null +++ b/state/templates_test.go @@ -0,0 +1,42 @@ +package state + +import ( + "github.com/netsampler/goflow2/v2/decoders/netflow" + "testing" +) + +func benchTemplatesAdd(ts netflow.NetFlowTemplateSystem, obs uint32, N int, b *testing.B) { + for n := 0; n <= N; n++ { + ts.AddTemplate(10, obs, uint16(n), n) + } +} + +func BenchmarkTemplatesAdd(b *testing.B) { + InitTemplates() + ts := CreateTemplateSystem("1") + b.Log("Creating", b.N, "templates") + benchTemplatesAdd(ts, uint32(b.N)%0xffff+1, b.N, b) +} + +func BenchmarkTemplatesAddGet(b *testing.B) { + InitTemplates() + ts := CreateTemplateSystem("2") + templates := 1000 + b.Log("Adding", templates, "templates") + benchTemplatesAdd(ts, 1, templates, b) + b.Log("Getting", b.N, "templates") + + for n := 0; n <= b.N; n++ { + data, err := ts.GetTemplate(10, 1, uint16(n%templates)) + if err != nil { + b.Fatal(err) + } + dataC, ok := data.(int) + if !ok { + b.Fatal("template not an integer") + } + if dataC != n%templates { + b.Fatal("different values", dataC, "!=", n%templates) + } + } +} From 580b06ee72ded71356131b7cd158cb3b2b7603d6 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 13:49:13 +0700 Subject: [PATCH 17/21] remove unused codes --- decoders/netflow/templates.go | 69 ------------------------------ decoders/netflow/templates_test.go | 39 ----------------- producer/proto/producer_nf.go | 41 ------------------ 3 files changed, 149 deletions(-) delete mode 100644 decoders/netflow/templates_test.go diff --git a/decoders/netflow/templates.go b/decoders/netflow/templates.go index 42be916b..777accc1 100644 --- a/decoders/netflow/templates.go +++ b/decoders/netflow/templates.go @@ -2,84 +2,15 @@ package netflow import ( "fmt" - "sync" ) var ( ErrorTemplateNotFound = fmt.Errorf("Error template not found") ) -type FlowBaseTemplateSet map[uint64]interface{} - -func templateKey(version uint16, obsDomainId uint32, templateId uint16) uint64 { - return (uint64(version) << 48) | (uint64(obsDomainId) << 16) | uint64(templateId) -} - // Store interface that allows storing, removing and retrieving template data type NetFlowTemplateSystem interface { RemoveTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error } - -func (ts *BasicTemplateSystem) GetTemplates() FlowBaseTemplateSet { - ts.templateslock.RLock() - tmp := ts.templates - ts.templateslock.RUnlock() - return tmp -} - -func (ts *BasicTemplateSystem) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error { - ts.templateslock.Lock() - defer ts.templateslock.Unlock() - - /*var templateId uint16 - switch templateIdConv := template.(type) { - case IPFIXOptionsTemplateRecord: - templateId = templateIdConv.TemplateId - case NFv9OptionsTemplateRecord: - templateId = templateIdConv.TemplateId - case TemplateRecord: - templateId = templateIdConv.TemplateId - }*/ - key := templateKey(version, obsDomainId, templateId) - ts.templates[key] = template - return nil -} - -func (ts *BasicTemplateSystem) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { - ts.templateslock.RLock() - defer ts.templateslock.RUnlock() - key := templateKey(version, obsDomainId, templateId) - if template, ok := ts.templates[key]; ok { - return template, nil - } - return nil, ErrorTemplateNotFound -} - -func (ts *BasicTemplateSystem) RemoveTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { - ts.templateslock.Lock() - defer ts.templateslock.Unlock() - - key := templateKey(version, obsDomainId, templateId) - if template, ok := ts.templates[key]; ok { - delete(ts.templates, key) - return template, nil - } - return nil, ErrorTemplateNotFound -} - -type BasicTemplateSystem struct { - templates FlowBaseTemplateSet - templateslock *sync.RWMutex -} - -// Creates a basic store for NetFlow and IPFIX templates. -// Everyting is stored in memory. -func CreateTemplateSystem() NetFlowTemplateSystem { - ts := &BasicTemplateSystem{ - templates: make(FlowBaseTemplateSet), - templateslock: &sync.RWMutex{}, - } - return ts -} diff --git a/decoders/netflow/templates_test.go b/decoders/netflow/templates_test.go deleted file mode 100644 index 4ee8f5e8..00000000 --- a/decoders/netflow/templates_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package netflow - -import ( - "testing" -) - -func benchTemplatesAdd(ts NetFlowTemplateSystem, obs uint32, N int, b *testing.B) { - for n := 0; n <= N; n++ { - ts.AddTemplate(10, obs, uint16(n), n) - } -} - -func BenchmarkTemplatesAdd(b *testing.B) { - ts := CreateTemplateSystem() - b.Log("Creating", b.N, "templates") - benchTemplatesAdd(ts, uint32(b.N)%0xffff+1, b.N, b) -} - -func BenchmarkTemplatesAddGet(b *testing.B) { - ts := CreateTemplateSystem() - templates := 1000 - b.Log("Adding", templates, "templates") - benchTemplatesAdd(ts, 1, templates, b) - b.Log("Getting", b.N, "templates") - - for n := 0; n <= b.N; n++ { - data, err := ts.GetTemplate(10, 1, uint16(n%templates)) - if err != nil { - b.Fatal(err) - } - dataC, ok := data.(int) - if !ok { - b.Fatal("template not an integer") - } - if dataC != n%templates { - b.Fatal("different values", dataC, "!=", n%templates) - } - } -} diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index ca3103c4..eda08d0a 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/binary" "fmt" - "sync" "time" "github.com/netsampler/goflow2/v2/decoders/netflow" @@ -18,46 +17,6 @@ type SamplingRateSystem interface { AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) } -type basicSamplingRateKey struct { - version uint16 - obsDomainId uint32 -} - -type basicSamplingRateSystem struct { - sampling map[basicSamplingRateKey]uint32 - samplinglock *sync.RWMutex -} - -func CreateSamplingSystem(key string) SamplingRateSystem { - ts := &basicSamplingRateSystem{ - sampling: make(map[basicSamplingRateKey]uint32), - samplinglock: &sync.RWMutex{}, - } - return ts -} - -func (s *basicSamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) { - s.samplinglock.Lock() - defer s.samplinglock.Unlock() - s.sampling[basicSamplingRateKey{ - version: version, - obsDomainId: obsDomainId, - }] = samplingRate -} - -func (s *basicSamplingRateSystem) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) { - s.samplinglock.RLock() - defer s.samplinglock.RUnlock() - if samplingRate, ok := s.sampling[basicSamplingRateKey{ - version: version, - obsDomainId: obsDomainId, - }]; ok { - return samplingRate, nil - } - - return 0, fmt.Errorf("sampling rate not found") -} - type SingleSamplingRateSystem struct { Sampling uint32 } From 6defb08e18a9b84a340fcea0d459b55a924d1b7a Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 14:04:01 +0700 Subject: [PATCH 18/21] move templates and sampling rate to the previous position --- cmd/goflow2/main.go | 11 +- decoders/netflow/netflow_test.go | 6 +- decoders/netflow/templates.go | 184 +++++++++++++++++ {state => decoders/netflow}/templates_test.go | 5 +- metrics/templates.go | 3 +- producer/proto/producer_nf.go | 74 +++++++ state/sampling_rate.go | 78 ------- state/templates.go | 190 ------------------ utils/templates/templates.go | 3 +- 9 files changed, 271 insertions(+), 283 deletions(-) rename {state => decoders/netflow}/templates_test.go (83%) delete mode 100644 state/sampling_rate.go delete mode 100644 state/templates.go diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go index 41421a4c..edeca6b1 100644 --- a/cmd/goflow2/main.go +++ b/cmd/goflow2/main.go @@ -38,7 +38,6 @@ import ( // core libraries "github.com/netsampler/goflow2/v2/metrics" - "github.com/netsampler/goflow2/v2/state" "github.com/netsampler/goflow2/v2/utils" "github.com/netsampler/goflow2/v2/utils/debug" @@ -122,7 +121,7 @@ func main() { log.Fatal(err) } } - flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, state.CreateSamplingSystem) + flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, protoproducer.CreateSamplingSystem) if err != nil { log.Fatal(err) } @@ -178,16 +177,16 @@ func main() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) - err = state.InitSamplingRate() + err = protoproducer.InitSamplingRate() if err != nil { log.Fatal(err) } - defer state.CloseSamplingRate() - err = state.InitTemplates() + defer protoproducer.CloseSamplingRate() + err = netflow.InitTemplates() if err != nil { log.Fatal(err) } - defer state.CloseTemplates() + defer netflow.CloseTemplates() var receivers []*utils.UDPReceiver var pipes []utils.FlowPipe diff --git a/decoders/netflow/netflow_test.go b/decoders/netflow/netflow_test.go index 8c940072..88e23664 100644 --- a/decoders/netflow/netflow_test.go +++ b/decoders/netflow/netflow_test.go @@ -8,7 +8,9 @@ import ( ) func TestDecodeNetFlowV9(t *testing.T) { - templates := CreateTemplateSystem() + err := InitTemplates() + assert.NoError(t, err) + templates := CreateTemplateSystem("TestDecodeNetFlowV9") // Decode a template template := []byte{ @@ -23,7 +25,7 @@ func TestDecodeNetFlowV9(t *testing.T) { } buf := bytes.NewBuffer(template) var decNfv9 NFv9Packet - err := DecodeMessageVersion(buf, templates, &decNfv9, nil) + err = DecodeMessageVersion(buf, templates, &decNfv9, nil) assert.Nil(t, err) assert.Equal(t, NFv9Packet{ diff --git a/decoders/netflow/templates.go b/decoders/netflow/templates.go index 777accc1..ca9b6560 100644 --- a/decoders/netflow/templates.go +++ b/decoders/netflow/templates.go @@ -1,11 +1,23 @@ package netflow import ( + "encoding/json" + "errors" + "flag" "fmt" + "net/url" + "reflect" + "strings" + "sync" + + "github.com/netsampler/goflow2/v2/state" ) var ( ErrorTemplateNotFound = fmt.Errorf("Error template not found") + StateTemplates = flag.String("state.netflow.templates", "memory://", fmt.Sprintf("Define state templates engine URL (available schemes: %s)", strings.Join(state.SupportedSchemes, ", "))) + templatesDB state.State[templatesKey, templatesValue] + templatesInitLock = new(sync.Mutex) ) // Store interface that allows storing, removing and retrieving template data @@ -14,3 +26,175 @@ type NetFlowTemplateSystem interface { GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error } + +type templatesKey struct { + Key string `json:"key"` + Version uint16 `json:"ver"` + ObsDomainId uint32 `json:"obs"` + TemplateID uint16 `json:"tid"` +} + +const ( + templateTypeTest = 0 + templateTypeTemplateRecord = 1 + templateTypeIPFIXOptionsTemplateRecord = 2 + templateTypeNFv9OptionsTemplateRecord = 3 +) + +type templatesValue struct { + TemplateType int `json:"ttype"` + Data interface{} `json:"data"` +} + +type templatesValueUnmarshal struct { + TemplateType int `json:"ttype"` + Data json.RawMessage `json:"data"` +} + +func (t *templatesValue) UnmarshalJSON(bytes []byte) error { + var v templatesValueUnmarshal + err := json.Unmarshal(bytes, &v) + if err != nil { + return err + } + t.TemplateType = v.TemplateType + switch v.TemplateType { + case templateTypeTest: + var data int + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + case templateTypeTemplateRecord: + var data TemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + case templateTypeIPFIXOptionsTemplateRecord: + var data IPFIXOptionsTemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + case templateTypeNFv9OptionsTemplateRecord: + var data NFv9OptionsTemplateRecord + err = json.Unmarshal(v.Data, &data) + if err != nil { + return err + } + t.Data = data + default: + return fmt.Errorf("unknown template type: %d", v.TemplateType) + } + return nil +} + +type NetflowTemplate struct { + key string +} + +func (t *NetflowTemplate) RemoveTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { + if v, err := templatesDB.Pop(templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + }); err != nil && errors.Is(err, state.ErrorKeyNotFound) { + return nil, ErrorTemplateNotFound + } else if err != nil { + return nil, err + } else { + return v.Data, nil + } +} + +func (t *NetflowTemplate) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { + if v, err := templatesDB.Get(templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + }); err != nil && errors.Is(err, state.ErrorKeyNotFound) { + return nil, ErrorTemplateNotFound + } else if err != nil { + return nil, err + } else { + return v.Data, nil + } +} + +func (t *NetflowTemplate) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error { + k := templatesKey{ + Key: t.key, + Version: version, + ObsDomainId: obsDomainId, + TemplateID: templateId, + } + var err error + switch templatec := template.(type) { + case TemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeTemplateRecord, + Data: templatec, + }) + case IPFIXOptionsTemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeIPFIXOptionsTemplateRecord, + Data: templatec, + }) + case NFv9OptionsTemplateRecord: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeNFv9OptionsTemplateRecord, + Data: templatec, + }) + case int: + err = templatesDB.Add(k, templatesValue{ + TemplateType: templateTypeTest, + Data: templatec, + }) + default: + return fmt.Errorf("unknown template type: %s", reflect.TypeOf(template).String()) + } + return err +} + +func CreateTemplateSystem(key string) NetFlowTemplateSystem { + ts := &NetflowTemplate{ + key: key, + } + return ts +} + +func InitTemplates() error { + templatesInitLock.Lock() + defer templatesInitLock.Unlock() + if templatesDB != nil { + return nil + } + templatesUrl, err := url.Parse(*StateTemplates) + if err != nil { + return err + } + if !templatesUrl.Query().Has("prefix") { + q := templatesUrl.Query() + q.Set("prefix", "goflow2:nf_templates:") + templatesUrl.RawQuery = q.Encode() + } + templatesDB, err = state.NewState[templatesKey, templatesValue](templatesUrl.String()) + return err +} + +func CloseTemplates() error { + templatesInitLock.Lock() + defer templatesInitLock.Unlock() + if templatesDB == nil { + return nil + } + err := templatesDB.Close() + templatesDB = nil + return err +} diff --git a/state/templates_test.go b/decoders/netflow/templates_test.go similarity index 83% rename from state/templates_test.go rename to decoders/netflow/templates_test.go index 648690ee..b77f785e 100644 --- a/state/templates_test.go +++ b/decoders/netflow/templates_test.go @@ -1,11 +1,10 @@ -package state +package netflow import ( - "github.com/netsampler/goflow2/v2/decoders/netflow" "testing" ) -func benchTemplatesAdd(ts netflow.NetFlowTemplateSystem, obs uint32, N int, b *testing.B) { +func benchTemplatesAdd(ts NetFlowTemplateSystem, obs uint32, N int, b *testing.B) { for n := 0; n <= N; n++ { ts.AddTemplate(10, obs, uint16(n), n) } diff --git a/metrics/templates.go b/metrics/templates.go index c35540c3..41367301 100644 --- a/metrics/templates.go +++ b/metrics/templates.go @@ -4,7 +4,6 @@ import ( "strconv" "github.com/netsampler/goflow2/v2/decoders/netflow" - "github.com/netsampler/goflow2/v2/state" "github.com/prometheus/client_golang/prometheus" ) @@ -16,7 +15,7 @@ type PromTemplateSystem struct { // A default Prometheus template generator function to be used by a pipe func NewDefaultPromTemplateSystem(key string) netflow.NetFlowTemplateSystem { - return NewPromTemplateSystem(key, state.CreateTemplateSystem(key)) + return NewPromTemplateSystem(key, netflow.CreateTemplateSystem(key)) } // Creates a Prometheus template system that wraps another template system. diff --git a/producer/proto/producer_nf.go b/producer/proto/producer_nf.go index eda08d0a..74c0528b 100644 --- a/producer/proto/producer_nf.go +++ b/producer/proto/producer_nf.go @@ -3,13 +3,18 @@ package protoproducer import ( "bytes" "encoding/binary" + "flag" "fmt" + "net/url" + "strings" + "sync" "time" "github.com/netsampler/goflow2/v2/decoders/netflow" "github.com/netsampler/goflow2/v2/decoders/utils" flowmessage "github.com/netsampler/goflow2/v2/pb" "github.com/netsampler/goflow2/v2/producer" + "github.com/netsampler/goflow2/v2/state" ) type SamplingRateSystem interface { @@ -28,6 +33,75 @@ func (s *SingleSamplingRateSystem) GetSamplingRate(version uint16, obsDomainId u return s.Sampling, nil } +var ( + StateSampling = flag.String("state.sampling", "memory://", fmt.Sprintf("Define state sampling rate engine URL (available schemes: %s)", strings.Join(state.SupportedSchemes, ", "))) + samplingRateDB state.State[samplingRateKey, uint32] + samplingRateInitLock = new(sync.Mutex) +) + +type samplingRateKey struct { + Key string `json:"key"` + Version uint16 `json:"ver"` + ObsDomainId uint32 `json:"obs"` +} + +type SamplingRate struct { + key string +} + +func (s *SamplingRate) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) { + return samplingRateDB.Get(samplingRateKey{ + Key: s.key, + Version: version, + ObsDomainId: obsDomainId, + }) +} + +func (s *SamplingRate) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) { + _ = samplingRateDB.Add(samplingRateKey{ + Key: s.key, + Version: version, + ObsDomainId: obsDomainId, + }, samplingRate) +} + +func CreateSamplingSystem(key string) SamplingRateSystem { + ts := &SamplingRate{ + key: key, + } + return ts +} + +func InitSamplingRate() error { + samplingRateInitLock.Lock() + defer samplingRateInitLock.Unlock() + if samplingRateDB != nil { + return nil + } + samplingUrl, err := url.Parse(*StateSampling) + if err != nil { + return err + } + if !samplingUrl.Query().Has("prefix") { + q := samplingUrl.Query() + q.Set("prefix", "goflow2:sampling_rate:") + samplingUrl.RawQuery = q.Encode() + } + samplingRateDB, err = state.NewState[samplingRateKey, uint32](samplingUrl.String()) + return err +} + +func CloseSamplingRate() error { + samplingRateInitLock.Lock() + defer samplingRateInitLock.Unlock() + if samplingRateDB == nil { + return nil + } + err := samplingRateDB.Close() + samplingRateDB = nil + return err +} + func NetFlowLookFor(dataFields []netflow.DataField, typeId uint16) (bool, interface{}) { for _, dataField := range dataFields { if dataField.Type == typeId { diff --git a/state/sampling_rate.go b/state/sampling_rate.go deleted file mode 100644 index bf856d87..00000000 --- a/state/sampling_rate.go +++ /dev/null @@ -1,78 +0,0 @@ -package state - -import ( - "flag" - "fmt" - "net/url" - "strings" - "sync" - - "github.com/netsampler/goflow2/v2/producer/proto" -) - -var StateSampling = flag.String("state.sampling", "memory://", fmt.Sprintf("Define state sampling rate engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) -var samplingRateDB State[samplingRateKey, uint32] -var samplingRateInitLock = new(sync.Mutex) - -type samplingRateKey struct { - Key string `json:"key"` - Version uint16 `json:"ver"` - ObsDomainId uint32 `json:"obs"` -} - -type SamplingRateSystem struct { - key string -} - -func (s *SamplingRateSystem) GetSamplingRate(version uint16, obsDomainId uint32) (uint32, error) { - return samplingRateDB.Get(samplingRateKey{ - Key: s.key, - Version: version, - ObsDomainId: obsDomainId, - }) -} - -func (s *SamplingRateSystem) AddSamplingRate(version uint16, obsDomainId uint32, samplingRate uint32) { - _ = samplingRateDB.Add(samplingRateKey{ - Key: s.key, - Version: version, - ObsDomainId: obsDomainId, - }, samplingRate) -} - -func CreateSamplingSystem(key string) protoproducer.SamplingRateSystem { - ts := &SamplingRateSystem{ - key: key, - } - return ts -} - -func InitSamplingRate() error { - samplingRateInitLock.Lock() - defer samplingRateInitLock.Unlock() - if samplingRateDB != nil { - return nil - } - samplingUrl, err := url.Parse(*StateSampling) - if err != nil { - return err - } - if !samplingUrl.Query().Has("prefix") { - q := samplingUrl.Query() - q.Set("prefix", "goflow2:sampling_rate:") - samplingUrl.RawQuery = q.Encode() - } - samplingRateDB, err = NewState[samplingRateKey, uint32](samplingUrl.String()) - return err -} - -func CloseSamplingRate() error { - samplingRateInitLock.Lock() - defer samplingRateInitLock.Unlock() - if samplingRateDB == nil { - return nil - } - err := samplingRateDB.Close() - samplingRateDB = nil - return err -} diff --git a/state/templates.go b/state/templates.go deleted file mode 100644 index 6c698f2e..00000000 --- a/state/templates.go +++ /dev/null @@ -1,190 +0,0 @@ -package state - -import ( - "encoding/json" - "errors" - "flag" - "fmt" - "net/url" - "reflect" - "strings" - "sync" - - "github.com/netsampler/goflow2/v2/decoders/netflow" -) - -var StateTemplates = flag.String("state.netflow.templates", "memory://", fmt.Sprintf("Define state templates engine URL (available schemes: %s)", strings.Join(SupportedSchemes, ", "))) -var templatesDB State[templatesKey, templatesValue] -var templatesInitLock = new(sync.Mutex) - -type templatesKey struct { - Key string `json:"key"` - Version uint16 `json:"ver"` - ObsDomainId uint32 `json:"obs"` - TemplateID uint16 `json:"tid"` -} - -const ( - templateTypeTest = 0 - templateTypeTemplateRecord = 1 - templateTypeIPFIXOptionsTemplateRecord = 2 - templateTypeNFv9OptionsTemplateRecord = 3 -) - -type templatesValue struct { - TemplateType int `json:"ttype"` - Data interface{} `json:"data"` -} - -type templatesValueUnmarshal struct { - TemplateType int `json:"ttype"` - Data json.RawMessage `json:"data"` -} - -func (t *templatesValue) UnmarshalJSON(bytes []byte) error { - var v templatesValueUnmarshal - err := json.Unmarshal(bytes, &v) - if err != nil { - return err - } - t.TemplateType = v.TemplateType - switch v.TemplateType { - case templateTypeTest: - var data int - err = json.Unmarshal(v.Data, &data) - if err != nil { - return err - } - t.Data = data - case templateTypeTemplateRecord: - var data netflow.TemplateRecord - err = json.Unmarshal(v.Data, &data) - if err != nil { - return err - } - t.Data = data - case templateTypeIPFIXOptionsTemplateRecord: - var data netflow.IPFIXOptionsTemplateRecord - err = json.Unmarshal(v.Data, &data) - if err != nil { - return err - } - t.Data = data - case templateTypeNFv9OptionsTemplateRecord: - var data netflow.NFv9OptionsTemplateRecord - err = json.Unmarshal(v.Data, &data) - if err != nil { - return err - } - t.Data = data - default: - return fmt.Errorf("unknown template type: %d", v.TemplateType) - } - return nil -} - -type TemplateSystem struct { - key string -} - -func (t *TemplateSystem) RemoveTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { - if v, err := templatesDB.Pop(templatesKey{ - Key: t.key, - Version: version, - ObsDomainId: obsDomainId, - TemplateID: templateId, - }); err != nil && errors.Is(err, ErrorKeyNotFound) { - return nil, netflow.ErrorTemplateNotFound - } else if err != nil { - return nil, err - } else { - return v.Data, nil - } -} - -func (t *TemplateSystem) GetTemplate(version uint16, obsDomainId uint32, templateId uint16) (interface{}, error) { - if v, err := templatesDB.Get(templatesKey{ - Key: t.key, - Version: version, - ObsDomainId: obsDomainId, - TemplateID: templateId, - }); err != nil && errors.Is(err, ErrorKeyNotFound) { - return nil, netflow.ErrorTemplateNotFound - } else if err != nil { - return nil, err - } else { - return v.Data, nil - } -} - -func (t *TemplateSystem) AddTemplate(version uint16, obsDomainId uint32, templateId uint16, template interface{}) error { - k := templatesKey{ - Key: t.key, - Version: version, - ObsDomainId: obsDomainId, - TemplateID: templateId, - } - var err error - switch templatec := template.(type) { - case netflow.TemplateRecord: - err = templatesDB.Add(k, templatesValue{ - TemplateType: templateTypeTemplateRecord, - Data: templatec, - }) - case netflow.IPFIXOptionsTemplateRecord: - err = templatesDB.Add(k, templatesValue{ - TemplateType: templateTypeIPFIXOptionsTemplateRecord, - Data: templatec, - }) - case netflow.NFv9OptionsTemplateRecord: - err = templatesDB.Add(k, templatesValue{ - TemplateType: templateTypeNFv9OptionsTemplateRecord, - Data: templatec, - }) - case int: - err = templatesDB.Add(k, templatesValue{ - TemplateType: templateTypeTest, - Data: templatec, - }) - default: - return fmt.Errorf("unknown template type: %s", reflect.TypeOf(template).String()) - } - return err -} - -func CreateTemplateSystem(key string) netflow.NetFlowTemplateSystem { - ts := &TemplateSystem{ - key: key, - } - return ts -} - -func InitTemplates() error { - templatesInitLock.Lock() - defer templatesInitLock.Unlock() - if templatesDB != nil { - return nil - } - templatesUrl, err := url.Parse(*StateTemplates) - if err != nil { - return err - } - if !templatesUrl.Query().Has("prefix") { - q := templatesUrl.Query() - q.Set("prefix", "goflow2:nf_templates:") - templatesUrl.RawQuery = q.Encode() - } - templatesDB, err = NewState[templatesKey, templatesValue](templatesUrl.String()) - return err -} - -func CloseTemplates() error { - templatesInitLock.Lock() - defer templatesInitLock.Unlock() - if templatesDB == nil { - return nil - } - err := templatesDB.Close() - templatesDB = nil - return err -} diff --git a/utils/templates/templates.go b/utils/templates/templates.go index 8259818a..b6038756 100644 --- a/utils/templates/templates.go +++ b/utils/templates/templates.go @@ -2,7 +2,6 @@ package templates import ( "github.com/netsampler/goflow2/v2/decoders/netflow" - "github.com/netsampler/goflow2/v2/state" ) // Function that Create Template Systems. @@ -11,5 +10,5 @@ type TemplateSystemGenerator func(key string) netflow.NetFlowTemplateSystem // Default template generator func DefaultTemplateGenerator(key string) netflow.NetFlowTemplateSystem { - return state.CreateTemplateSystem(key) + return netflow.CreateTemplateSystem(key) } From 04b6135c04ea40c6e74e89adcb404b50dcb07803 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 14:08:35 +0700 Subject: [PATCH 19/21] changes nits --- cmd/goflow2/main.go | 1 + decoders/netflow/templates_test.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go index edeca6b1..d695a9fc 100644 --- a/cmd/goflow2/main.go +++ b/cmd/goflow2/main.go @@ -121,6 +121,7 @@ func main() { log.Fatal(err) } } + flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, protoproducer.CreateSamplingSystem) if err != nil { log.Fatal(err) diff --git a/decoders/netflow/templates_test.go b/decoders/netflow/templates_test.go index b77f785e..cdb075c0 100644 --- a/decoders/netflow/templates_test.go +++ b/decoders/netflow/templates_test.go @@ -12,14 +12,14 @@ func benchTemplatesAdd(ts NetFlowTemplateSystem, obs uint32, N int, b *testing.B func BenchmarkTemplatesAdd(b *testing.B) { InitTemplates() - ts := CreateTemplateSystem("1") + ts := CreateTemplateSystem("BenchmarkTemplatesAdd") b.Log("Creating", b.N, "templates") benchTemplatesAdd(ts, uint32(b.N)%0xffff+1, b.N, b) } func BenchmarkTemplatesAddGet(b *testing.B) { InitTemplates() - ts := CreateTemplateSystem("2") + ts := CreateTemplateSystem("BenchmarkTemplatesAddGet") templates := 1000 b.Log("Adding", templates, "templates") benchTemplatesAdd(ts, 1, templates, b) From 52649721bfe8b8fc8fb5312824d38a1197743852 Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 14:44:24 +0700 Subject: [PATCH 20/21] state documentation --- README.md | 13 +++++++++++-- docs/state_storage.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 docs/state_storage.md diff --git a/README.md b/README.md index c42838b0..2969129f 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,9 @@ functions to marshal as JSON or text for instance. The `transport` provides different way of processing the message. Either sending it via Kafka or send it to a file (or stdout). +The `state` supports external storage for a way of synchronizing templates across multiple GoFlow2 +instances. + GoFlow2 is a wrapper of all the functions and chains thems. You can build your own collector using this base and replace parts: @@ -53,6 +56,7 @@ You can build your own collector using this base and replace parts: * Convert to another format (e.g: Cap'n Proto, Avro, instead of protobuf) * Decode different samples (e.g: not only IP networks, add MPLS) * Different metrics system (e.g: [OpenTelemetry](https://opentelemetry.io/)) +* Other external state storage (e.g: RDBMS, MongoDB, memcached) ### Protocol difference @@ -85,8 +89,8 @@ Production: * Convert to protobuf or json * Prints to the console/file * Sends to Kafka and partition - -Monitoring via Prometheus metrics +* Set up multiple GoFlow2 instances backed by the same external state storage +* Monitoring via Prometheus metrics ## Get started @@ -165,6 +169,11 @@ $ ./goflow2 -listen 'sflow://:6343?count=4,nfl://:2055' More information about workers and resource usage is avaialble on the [Performance page](/docs/performance.md). +When you have multiple GoFlow2 instances, it's important to enable external state storage. +```bash +$ ./goflow2 -state.netflow.templates redis://127.0.0.1:6379/0?prefix=nftemplate -state.sampling redis://127.0.0.1:6379/0?prefix=nfsampling +``` + ### Docker You can also run directly with a container: diff --git a/docs/state_storage.md b/docs/state_storage.md new file mode 100644 index 00000000..20e4f9ba --- /dev/null +++ b/docs/state_storage.md @@ -0,0 +1,28 @@ +# State Storage + +For protocols with template system (Netflow V9 and IPFIX), GoFlow2 stores the information on memory by default. +When using memory, you will lose the template and sampling rate information if GoFlow2 is restarted. So incoming +flows will fail to decode until the next template/option data is sent from the agent. + +However, you can use an external state storage to overcome this issue. With external storage, you will have these +benefits: +- Supports UDP per-packet load balancer (e.g. with F5 or Envoy Proxy) +- Pod/container auto-scaling to handle traffic surge +- Persistent state, GoFlow2 restarts won't need to wait template/option data + +## Memory +The default method for storing state. It's not synced across multiple GoFlow2 instances and lost on process restart. + +## Redis +The supported URL format for redis +is explained at [uri specifications](https://github.com/redis/redis-specifications/blob/master/uri/redis.txt). +GoFlow2 uses the key-value storage provided by redis for persistence, and pub-sub to broadcast any new template data +to other GoFlow2 instances. +GoFlow2 also have other query parameters specific for redis: +- prefix + - this will override key prefix and channel prefix for pubsub + - e.g. redis://127.0.0.1/0?prefix=goflow +- interval + - specify in seconds on how frequent we should re-retrieve values (in case the pubsub doesn't work for some reason). + defaults to `900` seconds, use `0` to disable + - e.g. redis://127.0.0.1/0?interval=0 From 8a66b096358b99caa217246bc41f40a75dd59f1b Mon Sep 17 00:00:00 2001 From: Muhammad Iqbal Alaydrus Date: Fri, 16 Feb 2024 14:47:04 +0700 Subject: [PATCH 21/21] state documentation --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2969129f..1e91047f 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ When you have multiple GoFlow2 instances, it's important to enable external stat ```bash $ ./goflow2 -state.netflow.templates redis://127.0.0.1:6379/0?prefix=nftemplate -state.sampling redis://127.0.0.1:6379/0?prefix=nfsampling ``` +Details available on [State page](/docs/state_storage.md). ### Docker