Inject contents of INI files into environment variables for templates and wrapped app

LICENSE.md

@@ -0,0 +1,7 @@
+Copyright (c) 2020 The KBase Project and its Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

exec.go

@@ -14,6 +14,22 @@ import (
 func runCmd(ctx context.Context, cancel context.CancelFunc, cmd string, args ...string) {
 	defer wg.Done()
 
+	if eGID >= 0 {
+		log.Printf("Setting effective gid to %d", eGID)
+		err := Setgid(eGID)
+		if err != nil {
+			log.Fatalf("Error while setting GID to %d: %s", eGID, err)
+		}
+	}
+
+	if eUID >= 0 {
+		log.Printf("Setting effective uid to %d", eUID)
+		err := Setuid(eUID)
+		if err != nil {
+			log.Fatalf("Error while setting UID to %d: %s", eUID, err)
+		}
+	}
+
 	process := exec.Command(cmd, args...)
 	process.Stdin = os.Stdin
 	process.Stdout = os.Stdout

main.go

@@ -1,8 +1,11 @@
 package main
 
 import (
+	"crypto/tls"
+	"errors"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -13,21 +16,26 @@ import (
 	"time"
 
 	"golang.org/x/net/context"
+	"gopkg.in/ini.v1"
 )
 
 const defaultWaitRetryInterval = time.Second
 
 type sliceVar []string
 type hostFlagsVar []string
 
+// Context is the type passed into the template renderer
 type Context struct {
 }
 
-type HttpHeader struct {
+// HTTPHeader this is an optional header passed on http checks
+type HTTPHeader struct {
 	name  string
 	value string
 }
 
+// Env is bound to the template rendering Context and returns the
+// environment variables passed to the program
 func (c *Context) Env() map[string]string {
 	env := make(map[string]string)
 	for _, i := range os.Environ() {
@@ -43,20 +51,28 @@ var (
 	poll         bool
 	wg           sync.WaitGroup
 
+	envFlag           string
+	multiline         bool
+	envSection        string
+	envHdrFlag        sliceVar
+	validateCert      bool
+	httpFileFlag      sliceVar
 	templatesFlag     sliceVar
 	templateDirsFlag  sliceVar
 	stdoutTailFlag    sliceVar
 	stderrTailFlag    sliceVar
 	headersFlag       sliceVar
 	delimsFlag        string
 	delims            []string
-	headers           []HttpHeader
+	headers           []HTTPHeader
 	urls              []url.URL
 	waitFlag          hostFlagsVar
 	waitRetryInterval time.Duration
 	waitTimeoutFlag   time.Duration
 	dependencyChan    chan struct{}
 	noOverwriteFlag   bool
+	eUID              int
+	eGID              int
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -114,8 +130,15 @@ func waitForDependencies() {
 			case "http", "https":
 				wg.Add(1)
 				go func(u url.URL) {
+					var tr = http.DefaultTransport
+					if !validateCert {
+						tr = &http.Transport{
+							TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+						}
+					}
 					client := &http.Client{
-						Timeout: waitTimeoutFlag,
+						Timeout:   waitTimeoutFlag,
+						Transport: tr,
 					}
 
 					defer wg.Done()
@@ -206,11 +229,83 @@ Arguments:
 	println(`For more information, see https://github.com/jwilder/dockerize`)
 }
 
+func getURL(url string, envHdrFlag []string) (iniFile []byte, err error) {
+
+	var resp *http.Response
+	var req *http.Request
+	var hdr string
+	var client *http.Client
+	var tr = http.DefaultTransport
+	// Define redirect handler to disallow redirects
+	var redir = func(req *http.Request, via []*http.Request) error {
+		return errors.New("Redirects disallowed")
+	}
+
+	if !validateCert {
+		tr = &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+	}
+	client = &http.Client{Transport: tr, CheckRedirect: redir}
+	req, err = http.NewRequest("GET", url, nil)
+	if err != nil {
+		// Weird problem with declaring client, bail
+		return
+	}
+
+	// Handle headers for request - are they headers or filepaths?
+	for _, h := range envHdrFlag {
+		if strings.Contains(h, ":") {
+			// This will break if path includes colon - don't use colons in path!
+			hdr = h
+		} else { // Treat this is a path to a secrets file containing header
+			var hdrFile []byte
+			hdrFile, err = ioutil.ReadFile(h)
+			if err != nil { // Could not read file, error out
+				return
+			}
+			hdr = string(hdrFile)
+		}
+		parts := strings.Split(hdr, ":")
+		if len(parts) != 2 {
+			log.Fatalf("Bad env-headers argument: %s. expected \"headerName: headerValue\"", hdr)
+		}
+		req.Header.Add(strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]))
+	}
+	resp, err = client.Do(req)
+	if err == nil && resp.StatusCode == 200 {
+		defer resp.Body.Close()
+		iniFile, err = ioutil.ReadAll(resp.Body)
+	} else if err == nil { // Request completed with unexpected HTTP status code, bail
+		err = errors.New(resp.Status)
+	}
+	return
+}
+
+func getINI(envFlag string, envHdrFlag []string) (iniFile []byte, err error) {
+
+	// See if envFlag parses like an absolute URL, if so use http, otherwise treat as filename
+	url, urlERR := url.ParseRequestURI(envFlag)
+	if urlERR == nil && url.IsAbs() {
+		iniFile, err = getURL(envFlag, envHdrFlag)
+	} else {
+		iniFile, err = ioutil.ReadFile(envFlag)
+	}
+	return
+}
+
 func main() {
 
 	flag.BoolVar(&version, "version", false, "show version")
 	flag.BoolVar(&poll, "poll", false, "enable polling")
-
+	flag.StringVar(&envFlag, "env", "", "Optional path to INI file for injecting env vars. Does not overwrite existing env vars")
+	flag.BoolVar(&multiline, "multiline", false, "enable parsing multiline INI entries in INI environment file")
+	flag.StringVar(&envSection, "env-section", "", "Optional section of INI file to use for loading env vars. Defaults to \"\"")
+	flag.Var(&envHdrFlag, "env-header", "Optional string or path to secrets file for http headers passed if -env or -httpFile are URLs")
+	flag.BoolVar(&validateCert, "validate-cert", true, "Verify SSL certs for https connections")
+	flag.IntVar(&eGID, "egid", -1, "Set the numeric group ID for the running program") // Check for -1 later to skip
+	flag.IntVar(&eUID, "euid", -1, "Set the numeric user id for the running program")
+	flag.Var(&httpFileFlag, "httpFile", "Source URL and dest path (http://blah.com/blahblah~/dest). Pulls file from URL using env-header for auth and writes file to destination. Use tilde to separate URL from destination file")
 	flag.Var(&templatesFlag, "template", "Template (/template:/dest). Can be passed multiple times. Does also support directories")
 	flag.BoolVar(&noOverwriteFlag, "no-overwrite", false, "Do not overwrite destination file if it already exists.")
 	flag.Var(&stdoutTailFlag, "stdout", "Tails a file to stdout. Can be passed multiple times")
@@ -234,6 +329,25 @@ func main() {
 		os.Exit(1)
 	}
 
+	if envFlag != "" {
+		iniFile, err := getINI(envFlag, envHdrFlag)
+		if err != nil {
+			log.Fatalf("unreadable INI file %s: %s", envFlag, err)
+		}
+		cfg, err := ini.LoadSources(ini.LoadOptions{AllowPythonMultilineValues: multiline}, iniFile)
+		if err != nil {
+			log.Fatalf("error parsing contents of %s as INI format: %s", envFlag, err)
+		}
+		envHash := cfg.Section(envSection).KeysHash()
+
+		for k, v := range envHash {
+			if _, ok := os.LookupEnv(k); !ok {
+				// log.Printf("Setting %s to %s", k, v)
+				os.Setenv(k, v)
+			}
+		}
+	}
+
 	if delimsFlag != "" {
 		delims = strings.Split(delimsFlag, ":")
 		if len(delims) != 2 {
@@ -261,13 +375,34 @@ func main() {
 			if len(parts) != 2 {
 				log.Fatalf(errMsg, headersFlag)
 			}
-			headers = append(headers, HttpHeader{name: strings.TrimSpace(parts[0]), value: strings.TrimSpace(parts[1])})
+			headers = append(headers, HTTPHeader{name: strings.TrimSpace(parts[0]), value: strings.TrimSpace(parts[1])})
 		} else {
 			log.Fatalf(errMsg, headersFlag)
 		}
 
 	}
 
+	for _, httpFile := range httpFileFlag {
+		delim := "~"
+		if strings.Contains(httpFile, delim) {
+			parts := strings.Split(httpFile, delim)
+			if len(parts) != 2 {
+				log.Fatalf("bad httpFile argument: %s. expected \"URL;/dest\"", httpFile)
+			}
+			srcURL, dest := parts[0], parts[1]
+			httpSrc, err := getURL(srcURL, envHdrFlag)
+			if err != nil {
+				log.Fatalf("unable to fetch contents of url %s, error: %s", srcURL, err)
+			}
+			err = ioutil.WriteFile(dest, httpSrc, 0644)
+			if err != nil {
+				log.Fatalf("unable to write httpFile contents to %s, error: %s", dest, err)
+			}
+		} else {
+			log.Fatalf("-httpFile switch missing %s delimiter: %s", delim, httpFile)
+		}
+	}
+
 	for _, t := range templatesFlag {
 		template, dest := t, ""
 		if strings.Contains(t, ":") {
@@ -296,6 +431,8 @@ func main() {
 
 	if flag.NArg() > 0 {
 		wg.Add(1)
+		// Drop privs if passed the euid or egid params
+
 		go runCmd(ctx, cancel, flag.Arg(0), flag.Args()[1:]...)
 	}
 

system.go

@@ -0,0 +1,26 @@
+package main
+
+// This has been cut/pasted from
+// https://github.com/opencontainers/runc/blob/master/libcontainer/system/syscall_linux_64.go
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+// Setuid sets the uid of the calling thread to the specified uid.
+func Setuid(uid int) (err error) {
+	_, _, e1 := unix.RawSyscall(unix.SYS_SETUID, uintptr(uid), 0, 0)
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}
+
+// Setgid sets the gid of the calling thread to the specified gid.
+func Setgid(gid int) (err error) {
+	_, _, e1 := unix.RawSyscall(unix.SYS_SETGID, uintptr(gid), 0, 0)
+	if e1 != 0 {
+		err = e1
+	}
+	return
+}

tail.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"time"
 
 	"github.com/hpcloud/tail"
 	"golang.org/x/net/context"
@@ -12,17 +13,25 @@ import (
 func tailFile(ctx context.Context, file string, poll bool, dest *os.File) {
 	defer wg.Done()
 
+	var isPipe bool
+	var errCount int
+	const maxErr = 30
+	const sleepDur = 2 * time.Second
+
 	s, err := os.Stat(file)
 	if err != nil {
-		log.Fatalf("unable to stat %s: %s", file, err)
+		log.Printf("Warning: unable to stat %s: %s", file, err)
+		isPipe = false
+	} else {
+		isPipe = s.Mode()&os.ModeNamedPipe != 0
 	}
 
 	t, err := tail.TailFile(file, tail.Config{
 		Follow: true,
 		ReOpen: true,
 		Poll:   poll,
 		Logger: tail.DiscardingLogger,
-		Pipe:   s.Mode()&os.ModeNamedPipe != 0,
+		Pipe:   isPipe,
 	})
 	if err != nil {
 		log.Fatalf("unable to tail %s: %s", file, err)
@@ -41,15 +50,19 @@ func tailFile(ctx context.Context, file string, poll bool, dest *os.File) {
 			return
 		// get the next log line and echo it out
 		case line := <-t.Lines:
-			if line == nil {
-				if t.Err() != nil {
-					log.Fatalf("unable to tail %s: %s", file, t.Err())
+			if line.Err != nil || (line == nil && t.Err() != nil) {
+				log.Printf("Warning: unable to tail %s: %s", file, t.Err())
+				errCount++
+				if errCount > maxErr {
+					log.Fatalf("Logged %d consecutive errors while tailing. Exiting", errCount)
 				}
+				time.Sleep(sleepDur)
+				continue
+			} else if line == nil {
 				return
-			} else if line.Err != nil {
-				log.Fatalf("unable to tail %s: %s", file, t.Err())
 			}
 			fmt.Fprintln(dest, line.Text)
+			errCount = 0 // Zero the error count
 		}
 	}
 }