Support BOOTSTRAP_SERVER_MAPPING and EXTERNAL_SERVER_MAPPING as env variables

Author: everesio

Gopkg.lock

@@ -8,7 +8,7 @@ VERSION       ?= $(shell git describe --tags --always --dirty)
 GOPKGS         = $(shell go list ./... | grep -v /vendor/)
 BUILD_FLAGS   ?=
 LDFLAGS       ?= -X github.com/grepplabs/kafka-proxy/config.Version=$(VERSION) -w -s
-TAG           ?= "v0.0.5"
+TAG           ?= "v0.0.6"
 GOARCH        ?= amd64
 GOOS          ?= linux
 

Makefile

@@ -34,11 +34,11 @@ See:
 
    Linux
 
-        curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.0.5/kafka-proxy_0.0.5_linux_amd64.tar.gz | tar xz
+        curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.0.6/kafka-proxy_0.0.6_linux_amd64.tar.gz | tar xz
 
    macOS
 
-        curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.0.5/kafka-proxy_0.0.5_darwin_amd64.tar.gz | tar xz
+        curl -Ls https://github.com/grepplabs/kafka-proxy/releases/download/v0.0.6/kafka-proxy_0.0.6_darwin_amd64.tar.gz | tar xz
 
 2. Move the binary in to your PATH.
 
@@ -148,6 +148,8 @@ See:
                        --tls-enable --tls-insecure-skip-verify \
                        --sasl-enable --sasl-username myuser --sasl-password mysecret
 
+    export BOOTSTRAP_SERVER_MAPPING="192.168.99.100:32401,0.0.0.0:32402 192.168.99.100:32402,0.0.0.0:32403" && kafka-proxy server
+
 ### Proxy authentication example
 
     make clean build plugin.auth-user && build/kafka-proxy server --proxy-listener-key-file "server-key.pem"  \

README.md

@@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -25,11 +26,13 @@ import (
 	localauth "github.com/grepplabs/kafka-proxy/plugin/local-auth/shared"
 	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/go-plugin"
+	"strings"
 
 	"github.com/grepplabs/kafka-proxy/pkg/registry"
 	// built-in plugins
 	_ "github.com/grepplabs/kafka-proxy/pkg/libs/googleid-info"
 	_ "github.com/grepplabs/kafka-proxy/pkg/libs/googleid-provider"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -48,10 +51,10 @@ var Server = &cobra.Command{
 		if err := c.InitSASLCredentials(); err != nil {
 			return err
 		}
-		if err := c.InitBootstrapServers(bootstrapServersMapping); err != nil {
+		if err := c.InitBootstrapServers(getOrStringSlice(bootstrapServersMapping, "bootstrap-server-mapping")); err != nil {
 			return err
 		}
-		if err := c.InitExternalServers(externalServersMapping); err != nil {
+		if err := c.InitExternalServers(getOrStringSlice(externalServersMapping, "external-server-mapping")); err != nil {
 			return err
 		}
 		if err := c.Validate(); err != nil {
@@ -62,11 +65,21 @@ var Server = &cobra.Command{
 	Run: Run,
 }
 
+func getOrStringSlice(value []string, key string) []string {
+	if len(bootstrapServersMapping) != 0 {
+		return value
+	}
+	return viper.GetStringSlice(key)
+}
+
 func init() {
+	initFlags()
+}
+
+func initFlags() {
 	// proxy
 	Server.Flags().StringVar(&c.Proxy.DefaultListenerIP, "default-listener-ip", "127.0.0.1", "Default listener IP")
 	Server.Flags().StringArrayVar(&bootstrapServersMapping, "bootstrap-server-mapping", []string{}, "Mapping of Kafka bootstrap server address to local address (host:port,host:port(,advhost:advport))")
-	Server.MarkFlagRequired("bootstrap-server-mapping")
 	Server.Flags().StringArrayVar(&externalServersMapping, "external-server-mapping", []string{}, "Mapping of Kafka server address to external address (host:port,host:port). A listener for the external address is not started")
 	Server.Flags().BoolVar(&c.Proxy.DisableDynamicListeners, "dynamic-listeners-disable", false, "Disable dynamic listeners.")
 
@@ -151,6 +164,9 @@ func init() {
 
 	// Connect through Socks5 or HTTP CONNECT to Kafka
 	Server.Flags().StringVar(&c.ForwardProxy.Url, "forward-proxy", "", "URL of the forward proxy. Supported schemas are socks5 and http")
+
+	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
+	viper.AutomaticEnv() // read in environment variables that match
 }
 
 func Run(_ *cobra.Command, _ []string) {

cmd/kafka-proxy/server.go

@@ -0,0 +1,76 @@
+package server
+
+import (
+	"github.com/grepplabs/kafka-proxy/config"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func setupBootstrapServersMappingTest() {
+	Server.ResetFlags()
+	c = new(config.Config)
+	initFlags()
+	os.Setenv("BOOTSTRAP_SERVER_MAPPING", "")
+}
+
+func TestBootstrapServersMappingFromFlags(t *testing.T) {
+	setupBootstrapServersMappingTest()
+
+	args := []string{"cobra.test",
+		"--bootstrap-server-mapping", "192.168.99.100:32401,0.0.0.0:32401",
+		"--bootstrap-server-mapping", "192.168.99.100:32402,0.0.0.0:32402",
+		"--bootstrap-server-mapping", "kafka-2.example.com:9092,0.0.0.0:32403,kafka-2.grepplabs.com:9092",
+	}
+
+	Server.ParseFlags(args)
+	err := Server.PreRunE(nil, args)
+	a := assert.New(t)
+	a.Nil(err)
+	a.Len(c.Proxy.BootstrapServers, 3)
+
+	a.Equal(c.Proxy.BootstrapServers[0].BrokerAddress, "192.168.99.100:32401")
+	a.Equal(c.Proxy.BootstrapServers[0].ListenerAddress, "0.0.0.0:32401")
+	a.Equal(c.Proxy.BootstrapServers[0].AdvertisedAddress, "0.0.0.0:32401")
+
+	a.Equal(c.Proxy.BootstrapServers[1].BrokerAddress, "192.168.99.100:32402")
+	a.Equal(c.Proxy.BootstrapServers[1].ListenerAddress, "0.0.0.0:32402")
+	a.Equal(c.Proxy.BootstrapServers[1].AdvertisedAddress, "0.0.0.0:32402")
+
+	a.Equal(c.Proxy.BootstrapServers[2].BrokerAddress, "kafka-2.example.com:9092")
+	a.Equal(c.Proxy.BootstrapServers[2].ListenerAddress, "0.0.0.0:32403")
+	a.Equal(c.Proxy.BootstrapServers[2].AdvertisedAddress, "kafka-2.grepplabs.com:9092")
+
+}
+
+func TestBootstrapServersMappingFromEnv(t *testing.T) {
+	setupBootstrapServersMappingTest()
+
+	os.Setenv("BOOTSTRAP_SERVER_MAPPING", "192.168.99.100:32404,0.0.0.0:32404 kafka-5.example.com:9092,0.0.0.0:32405,kafka-5.grepplabs.com:9092")
+
+	var args []string
+	Server.ParseFlags(args)
+	err := Server.PreRunE(nil, args)
+	a := assert.New(t)
+	a.Nil(err)
+	a.Len(c.Proxy.BootstrapServers, 2)
+
+	a.Equal(c.Proxy.BootstrapServers[0].BrokerAddress, "192.168.99.100:32404")
+	a.Equal(c.Proxy.BootstrapServers[0].ListenerAddress, "0.0.0.0:32404")
+	a.Equal(c.Proxy.BootstrapServers[0].AdvertisedAddress, "0.0.0.0:32404")
+
+	a.Equal(c.Proxy.BootstrapServers[1].BrokerAddress, "kafka-5.example.com:9092")
+	a.Equal(c.Proxy.BootstrapServers[1].ListenerAddress, "0.0.0.0:32405")
+	a.Equal(c.Proxy.BootstrapServers[1].AdvertisedAddress, "kafka-5.grepplabs.com:9092")
+
+}
+
+func TestEmptyBootstrapServersMapping(t *testing.T) {
+	setupBootstrapServersMappingTest()
+
+	var args []string
+	Server.ParseFlags(args)
+	err := Server.PreRunE(nil, args)
+	a := assert.New(t)
+	a.Error(err, "list of bootstrap-server-mapping must not be empty")
+}