Updated the calculation of the number of possible values to generation

Author: yel0k

go.mod

@@ -5,12 +5,14 @@ go 1.23.8
 require (
 	github.com/apache/arrow-go/v18 v18.2.0
 	github.com/charmbracelet/huh/spinner v0.0.0-20250203114958-f07ae1af69ae
+	github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/ilyakaznacheev/cleanenv v1.5.0
 	github.com/labstack/echo/v4 v4.13.3
 	github.com/manifoldco/promptui v0.9.0
 	github.com/moby/term v0.5.2
+	github.com/otaviokr/topological-sort v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/sashabaranov/go-openai v1.36.1
 	github.com/spf13/afero v1.12.0
@@ -36,9 +38,7 @@ require (
 	github.com/charmbracelet/x/term v0.2.1 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
-	github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v25.2.10+incompatible // indirect
@@ -60,7 +60,6 @@ require (
 	github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
-	github.com/otaviokr/topological-sort v1.1.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect

go.sum

@@ -41,23 +41,20 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
-github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
 github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3 h1:fmFk0Wt3bBxxwZnu48jqMdaOR/IZ4vdtJFuaFV8MpIE=
 github.com/flosch/pongo2 v0.0.0-20200913210552-0d938eb266f3/go.mod h1:bJWSKrZyQvfTnb2OudyUjurSG4/edverV7n82+K3JiM=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
-github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
-github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -79,7 +76,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -88,8 +84,6 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
 github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
-github.com/heimdalr/dag v1.5.0 h1:hqVtijvY776P5OKP3QbdVBRt3Xxq6BYopz3XgklsGvo=
-github.com/heimdalr/dag v1.5.0/go.mod h1:lthekrHl01dddmzqyBQ1YZbi7XcVGGzjFo0jIky5knc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ilyakaznacheev/cleanenv v1.5.0 h1:0VNZXggJE2OYdXE87bfSSwGxeiGt9moSR2lOrsHHvr4=
 github.com/ilyakaznacheev/cleanenv v1.5.0/go.mod h1:a5aDzaJrLCQZsazHol1w8InnDcOX0OColm64SlIi6gk=
@@ -141,12 +135,15 @@ github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54
 github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54=
 github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
 github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
 github.com/otaviokr/topological-sort v1.1.0 h1:BrWj/bLOo9aZFUi0YN2/s4P/GRe2PSmb8cyX4w1ysNg=
 github.com/otaviokr/topological-sort v1.1.0/go.mod h1:77ZaKUg7Ir1nL6DPwEIQFm9iH2OS5xxVWvzZ8xPTCFg=
@@ -278,10 +275,12 @@ gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/generator/common/utils.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -357,3 +358,23 @@ func CtxClosed(ctx context.Context) bool {
 		return false
 	}
 }
+
+func ExtractValuesFromTemplate(template string) []string {
+	re := regexp.MustCompile(`{{\s*([^\s|(){}]+)[^}]*}}`)
+	matches := re.FindAllStringSubmatch(template, -1)
+
+	values := make([]string, 0, len(matches))
+
+	for _, match := range matches {
+		expr := match[0]
+		val := match[1]
+
+		if strings.Contains(expr, "(") && strings.Contains(expr, ")") {
+			continue
+		}
+
+		values = append(values, val)
+	}
+
+	return values
+}

internal/generator/common/utils_test.go

@@ -697,3 +697,50 @@ func TestWalkWithFilter(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) { testFunc(t, tc) })
 	}
 }
+
+func TestExtractValuesFromTemplate(t *testing.T) {
+	type testCase struct {
+		name     string
+		template string
+		expected []string
+	}
+
+	testCases := []testCase{
+		{
+			name:     "Empty template",
+			template: "",
+			expected: []string{},
+		},
+		{
+			name:     "Valid template",
+			template: "{{ foo }}.{{boo}}",
+			expected: []string{"foo", "boo"},
+		},
+		{
+			name:     "Template with filters",
+			template: "{{ foo | upper | lower }}",
+			expected: []string{"foo"},
+		},
+		{
+			name:     "Template with functions",
+			template: "{{ upper('foo') | lower }}@{{ boo }}",
+			expected: []string{"boo"},
+		},
+		{
+			name:     "Invalid template",
+			template: "{_{ foo }}",
+			expected: []string{},
+		},
+	}
+
+	testFunc := func(t *testing.T, tc testCase) {
+		t.Helper()
+
+		actual := ExtractValuesFromTemplate(tc.template)
+		require.Equal(t, tc.expected, actual)
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) { testFunc(t, tc) })
+	}
+}

internal/generator/models/common.go

@@ -2,15 +2,15 @@ package models
 
 import (
 	"encoding/json"
-	"github.com/otaviokr/topological-sort/toposort"
+	"github.com/tarantool/sdvg/internal/generator/common"
 	"io"
 	"os"
 	"path/filepath"
 	"reflect"
-	"regexp"
 	"strings"
 
 	"github.com/ilyakaznacheev/cleanenv"
+	"github.com/otaviokr/topological-sort/toposort"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 )
@@ -122,45 +122,24 @@ func parseErrsToString(errs []error) string {
 	return sb.String()
 }
 
-func TopologicalSort(columns []*Column) ([]string, error) {
+func topologicalSort(columns []*Column) ([]string, error) {
 	graph := make(map[string][]string)
 	for _, c := range columns {
 		graph[c.Name] = make([]string, 0)
 
-		if c.Type != "string" {
-			continue
-		}
-
 		for _, r := range c.Ranges {
-			if r.StringParams.Template == "" {
+			if r.StringParams == nil || r.StringParams.Template == "" {
 				continue
 			}
 
-			graph[c.Name] = extractValuesFromTemplate(r.StringParams.Template)
+			graph[c.Name] = common.ExtractValuesFromTemplate(r.StringParams.Template)
 		}
 	}
 
 	sortedVertexes, err := toposort.ReverseTarjan(graph)
 	if err != nil {
-		return nil, err
+		return nil, errors.New(err.Error())
 	}
 
 	return sortedVertexes, nil
 }
-
-func extractValuesFromTemplate(template string) []string {
-	re := regexp.MustCompile(`{{\s*([^}]+)\s*}}`)
-	matches := re.FindAllStringSubmatch(template, -1)
-
-	var values []string
-	for _, match := range matches {
-		expr := match[1]
-
-		parts := regexp.MustCompile(`\s*\|\s*|\s+`).Split(expr, -1)
-		if len(parts) > 0 && parts[0] != "" && !strings.Contains(parts[0], "(") {
-			values = append(values, parts[0])
-		}
-	}
-
-	return values
-}

internal/generator/models/common_test.go

@@ -1,56 +1,10 @@
 package models
 
 import (
-	"github.com/stretchr/testify/require"
 	"testing"
-)
-
-func TestExtractValuesFromTemplate(t *testing.T) {
-	type testCase struct {
-		name     string
-		template string
-		expected []string
-	}
-
-	testCases := []testCase{
-		{
-			name:     "Empty template",
-			template: "",
-			expected: nil,
-		},
-		{
-			name:     "Valid template",
-			template: "{{ foo }}.{{boo}}",
-			expected: []string{"foo", "boo"},
-		},
-		{
-			name:     "Template with filters",
-			template: "{{ foo | upper | lower }}",
-			expected: []string{"foo"},
-		},
-		{
-			name:     "Template with functions",
-			template: "{{ upper('foo') | lower }}@{{ boo }}",
-			expected: []string{"boo"},
-		},
-		{
-			name:     "Invalid template",
-			template: "{_{ foo }}",
-			expected: nil,
-		},
-	}
 
-	testFunc := func(t *testing.T, tc testCase) {
-		t.Helper()
-
-		actual := extractValuesFromTemplate(tc.template)
-		require.Equal(t, tc.expected, actual)
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) { testFunc(t, tc) })
-	}
-}
+	"github.com/stretchr/testify/require"
+)
 
 func TestTopologicalSort(t *testing.T) {
 	type testCase struct {
@@ -152,7 +106,7 @@ func TestTopologicalSort(t *testing.T) {
 	testFunc := func(t *testing.T, tc testCase) {
 		t.Helper()
 
-		actual, err := TopologicalSort(tc.columns)
+		actual, err := topologicalSort(tc.columns)
 		require.Equal(t, tc.wantErr, err != nil)
 		require.Equal(t, tc.expected, actual)
 	}

internal/generator/models/generator_model.go

@@ -30,8 +30,9 @@ type Model struct {
 	RowsPerFile   uint64  `backup:"true"      json:"rows_per_file" yaml:"rows_per_file"`
 	ModelDir      string  `backup:"true"      json:"model_dir"     yaml:"model_dir"`
 	// The columns from the partitioning key with PartitionColumn.WriteToOutput == false, must be at the end of slice.
-	Columns          []*Column          `backup:"true" json:"columns"           yaml:"columns"`
-	PartitionColumns []*PartitionColumn `backup:"true" json:"partition_columns" yaml:"partition_columns"`
+	Columns                 []*Column `backup:"true" json:"columns" yaml:"columns"`
+	ColumnsTopologicalOrder []string
+	PartitionColumns        []*PartitionColumn `backup:"true" json:"partition_columns" yaml:"partition_columns"`
 }
 
 // PartitionColumn type is used to describe partition parameters for column.
@@ -80,6 +81,13 @@ func (m *Model) Parse() error {
 
 	m.shiftColumnsToEnd(nonWriteableColumns)
 
+	sortedColumns, err := topologicalSort(m.Columns)
+	if err != nil {
+		return errors.WithMessage(err, "failed to sorting columns by dependencies")
+	}
+
+	m.ColumnsTopologicalOrder = sortedColumns
+
 	return nil
 }
 

internal/generator/models/models_test.go

@@ -1197,6 +1197,9 @@ parquet params:
 			expectedModel := tc.expected.Models[modelName]
 			gotModel := cfg.Models[modelName]
 
+			// skip ColumnsTopologicalOrder check
+			expectedModel.ColumnsTopologicalOrder = gotModel.ColumnsTopologicalOrder
+
 			for columnName := range expectedModel.Columns {
 				expectedColumn := expectedModel.Columns[columnName]
 				gotColumn := gotModel.Columns[columnName]