Merge pull request #28 from semaphoreci/env

Substitute env vars in Docker Images and change_in expressions.

Author: shiroyasha

.semaphore/semaphore.yml

@@ -58,7 +58,11 @@ blocks:
                 - test/e2e/change_in_excluded_paths.rb
                 - test/e2e/change_in_glob.rb
                 - test/e2e/change_in_invalid_when.rb
+                - test/e2e/change_in_java_vs_javascript_clash.rb
+                - test/e2e/change_in_large_commit_diff.rb
+                - test/e2e/change_in_large_commit_diff_on_default_branch.rb
                 - test/e2e/change_in_missing_branch.rb
+                - test/e2e/change_in_multiple_functions_in_one_when.rb
                 - test/e2e/change_in_multiple_paths.rb
                 - test/e2e/change_in_on_forked_prs.rb
                 - test/e2e/change_in_on_prs.rb
@@ -68,9 +72,8 @@ blocks:
                 - test/e2e/change_in_relative_paths.rb
                 - test/e2e/change_in_simple.rb
                 - test/e2e/change_in_with_default_branch.rb
-                - test/e2e/change_in_java_vs_javascript_clash.rb
-                - test/e2e/change_in_large_commit_diff.rb
-                - test/e2e/change_in_large_commit_diff_on_default_branch.rb
+                - test/e2e/env_simple.rb
+                - test/e2e/env_vars_in_docker_images.rb
                 - test/e2e/when_conditions_without_change_in.rb
 
           commands:

pkg/cli/evaluate.go

@@ -34,6 +34,9 @@ var evaluateChangeInCmd = &cobra.Command{
 		ppl, err := pipelines.LoadFromFile(input)
 		check(err)
 
+		err = ppl.SubstituteEnvVarsInDockerImages()
+		check(err)
+
 		err = ppl.EvaluateChangeIns()
 		check(err)
 

pkg/pipelines/model.go

@@ -2,10 +2,13 @@ package pipelines
 
 import (
 	"encoding/json"
+	"os"
+	"strconv"
 	"time"
 
 	gabs "github.com/Jeffail/gabs/v2"
 	"github.com/ghodss/yaml"
+	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
 )
 
 type Pipeline struct {
@@ -17,16 +20,61 @@ func n() int64 {
 	return time.Now().UnixNano() / int64(time.Millisecond)
 }
 
-func (p *Pipeline) UpdateWhenExpression(path []string, value string) error {
+func (p *Pipeline) UpdateString(path []string, value string) error {
 	_, err := p.raw.Set(value, path...)
 
 	return err
 }
 
+func (p *Pipeline) GetStringValueFrom(path []string) (string, bool) {
+	val, ok := p.raw.Search(path...).Data().(string)
+	return val, ok
+}
+
 func (p *Pipeline) EvaluateChangeIns() error {
 	return newWhenEvaluator(p).Run()
 }
 
+func (p *Pipeline) SubstituteEnvVarsInDockerImages() error {
+	consolelogger.Info("Expanding environment variables in YAML file")
+	consolelogger.EmptyLine()
+
+	containers := p.raw.Search("agent", "containers").Children()
+
+	for containerIndex := range containers {
+		path := []string{"agent", "containers", strconv.Itoa(containerIndex), "image"}
+
+		p.expandEnvIfExists(path)
+	}
+
+	for blockIndex := range p.Blocks() {
+		path := []string{"blocks", strconv.Itoa(blockIndex), "agent", "containers"}
+
+		containers := p.raw.Search(path...).Children()
+
+		for containerIndex := range containers {
+			path := append(path, []string{strconv.Itoa(containerIndex), "image"}...)
+
+			p.expandEnvIfExists(path)
+		}
+	}
+
+	return nil
+}
+
+func (p *Pipeline) expandEnvIfExists(path []string) {
+	if value, ok := p.GetStringValueFrom(path); ok {
+		newValue := os.ExpandEnv(value)
+
+		consolelogger.Infof("Expanding env vars in %+v\n", path)
+		consolelogger.Infof("Original: '%s'\n", value)
+		consolelogger.Infof("Expanded: '%s'\n", newValue)
+		consolelogger.EmptyLine()
+
+		p.UpdateString(path, newValue)
+	}
+}
+
 func (p *Pipeline) Blocks() []*gabs.Container {
 	return p.raw.Search("blocks").Children()
 }

pkg/pipelines/when_evaluator.go

@@ -55,7 +55,7 @@ func (e *whenEvaluator) Run() error {
 
 func (e *whenEvaluator) updatePipeline() error {
 	for index := range e.results {
-		err := e.pipeline.UpdateWhenExpression(e.list[index].Path, e.results[index])
+		err := e.pipeline.UpdateString(e.list[index].Path, e.results[index])
 
 		if err != nil {
 			return err

pkg/when/env/env.go

@@ -0,0 +1,36 @@
+package env
+
+import (
+	"fmt"
+	"os"
+
+	gabs "github.com/Jeffail/gabs/v2"
+	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
+)
+
+type Function struct {
+	VarName string
+}
+
+func Parse(ast *gabs.Container) (*Function, error) {
+	firstArg := ast.Search("params", "0")
+
+	if !firstArg.Exists() {
+		return nil, fmt.Errorf("variable name not found in env function")
+	}
+
+	varName, ok := firstArg.Data().(string)
+	if !ok {
+		return nil, fmt.Errorf("invalid variable name in env function")
+	}
+
+	return &Function{VarName: varName}, nil
+}
+
+func Eval(fun *Function) (string, error) {
+	value := os.Getenv(fun.VarName)
+
+	consolelogger.Infof("Result: '%+v'\n", value)
+
+	return value, nil
+}

pkg/when/expression.go

@@ -4,6 +4,7 @@ import (
 	gabs "github.com/Jeffail/gabs/v2"
 	consolelogger "github.com/semaphoreci/spc/pkg/consolelogger"
 	changein "github.com/semaphoreci/spc/pkg/when/changein"
+	env "github.com/semaphoreci/spc/pkg/when/env"
 	whencli "github.com/semaphoreci/spc/pkg/when/whencli"
 )
 
@@ -17,7 +18,22 @@ type WhenExpression struct {
 
 func (w *WhenExpression) Eval() error {
 	for _, requirment := range w.ListChangeInFunctions(w.Requirments) {
-		result, err := w.EvalFunction(requirment)
+		result, err := w.EvaluateChangeInFunction(requirment)
+		if err != nil {
+			return err
+		}
+
+		input := map[string]interface{}{}
+		input["name"] = w.functionName(requirment)
+		input["params"] = w.functionParams(requirment)
+		input["result"] = result
+
+		w.ReduceInputs.Keywords = map[string]interface{}{}
+		w.ReduceInputs.Functions = append(w.ReduceInputs.Functions, input)
+	}
+
+	for _, requirment := range w.ListEnvFunctions(w.Requirments) {
+		result, err := w.EvaluateEnvFunction(requirment)
 		if err != nil {
 			return err
 		}
@@ -46,6 +62,18 @@ func (w *WhenExpression) ListChangeInFunctions(requirments *gabs.Container) []*g
 	return result
 }
 
+func (w *WhenExpression) ListEnvFunctions(requirments *gabs.Container) []*gabs.Container {
+	result := []*gabs.Container{}
+
+	for _, input := range requirments.Children() {
+		if w.IsEnvFunction(input) {
+			result = append(result, input)
+		}
+	}
+
+	return result
+}
+
 func (w *WhenExpression) IsChangeInFunction(input *gabs.Container) bool {
 	elType := input.Search("type").Data().(string)
 	if elType != "fun" {
@@ -59,7 +87,20 @@ func (w *WhenExpression) IsChangeInFunction(input *gabs.Container) bool {
 	return true
 }
 
-func (w *WhenExpression) EvalFunction(input *gabs.Container) (bool, error) {
+func (w *WhenExpression) IsEnvFunction(input *gabs.Container) bool {
+	elType := input.Search("type").Data().(string)
+	if elType != "fun" {
+		return false
+	}
+
+	if w.functionName(input) != "env" {
+		return false
+	}
+
+	return true
+}
+
+func (w *WhenExpression) EvaluateChangeInFunction(input *gabs.Container) (bool, error) {
 	consolelogger.EmptyLine()
 
 	consolelogger.Infof("%s(%+v)\n", w.functionName(input), w.functionParams(input))
@@ -72,6 +113,19 @@ func (w *WhenExpression) EvalFunction(input *gabs.Container) (bool, error) {
 	return changein.Eval(fun)
 }
 
+func (w *WhenExpression) EvaluateEnvFunction(input *gabs.Container) (string, error) {
+	consolelogger.EmptyLine()
+
+	consolelogger.Infof("%s(%+v)\n", w.functionName(input), w.functionParams(input))
+
+	fun, err := env.Parse(input)
+	if err != nil {
+		return "", err
+	}
+
+	return env.Eval(fun)
+}
+
 func (w *WhenExpression) functionName(input *gabs.Container) string {
 	return input.Search("name").Data().(string)
 }

test/e2e/env_simple.rb

@@ -0,0 +1,88 @@
+# rubocop:disable all
+
+require_relative "../e2e"
+require 'yaml'
+
+pipeline = %q(
+version: v1.0
+name: Test
+agent:
+  machine:
+    type: e1-standard-2
+
+blocks:
+  - name: Test1
+    run:
+      when: "branch = 'master' or env('SEMAPHORE_GIT_PR_NAME') =~ '^docs:'"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+
+  - name: Test2
+    run:
+      when: "branch = 'master' or env('SEMAPHORE_GIT_PR_NAME') =~ '^feature:'"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+
+  - name: Test3
+    run:
+      when: "branch = 'master' or env('NON_EXISTENT_ENV_VAR') =~ '^feature:'"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+)
+
+origin = TestRepoForChangeIn.setup()
+
+origin.add_file('.semaphore/semaphore.yml', pipeline)
+origin.commit!("Bootstrap")
+
+repo = origin.clone_local_copy(branch: "master")
+repo.run(%{
+  export SEMAPHORE_GIT_PR_NAME="docs: Change in deployment order"
+
+  #{spc} evaluate change-in --input .semaphore/semaphore.yml --output /tmp/output.yml --logs /tmp/logs.yml
+})
+
+assert_eq(YAML.load_file('/tmp/output.yml'), YAML.load(%{
+version: v1.0
+name: Test
+agent:
+  machine:
+    type: e1-standard-2
+
+blocks:
+  - name: Test1
+    run:
+      when: "(branch = 'master') or true"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+
+  - name: Test2
+    run:
+      when: "(branch = 'master') or false"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+
+  - name: Test3
+    run:
+      when: "(branch = 'master') or false"
+    task:
+      jobs:
+        - name: Hello
+          commands:
+            - echo "Hello World"
+}))