diff --git a/.golangci.yml b/.golangci.yml index 7356d241..61a05971 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -73,7 +73,6 @@ linters: govet: enable: - nilness - - shadow lll: line-length: 140 misspell: diff --git a/README.md b/README.md index edc49b44..cb5b10de 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,42 @@ dmt lint /path/to/module1 /path/to/module2 /path/to/module3 Each directory is processed as a separate execution, and results are displayed for each directory individually. -### Gen +### Bootstrap -Generate some automatic rules for you module -\ +Bootstrap a new Deckhouse module from template: + +```shell +dmt bootstrap my-module-name +``` + +This command will: +- Download the official Deckhouse module template +- Extract it to the current directory (or specified directory) +- Replace template placeholders with your module name +- Configure CI/CD files based on your chosen platform + +#### Options + +- `--pipeline, -p`: Choose CI/CD platform (`github` or `gitlab`, default: `github`) +- `--directory, -d`: Specify target directory (default: current directory) +- `--repository-url, -r`: Use custom module template repository URL + +#### Examples + +Bootstrap a GitHub module: +```shell +dmt bootstrap my-awesome-module --pipeline github +``` + +Bootstrap a GitLab module in specific directory: +```shell +dmt bootstrap my-module --pipeline gitlab --directory ./modules/my-module +``` + +Use custom template repository: +```shell +dmt bootstrap my-module --repository-url https://github.com/myorg/custom-template/archive/main.zip +``` ## Linters list diff --git a/cmd/dmt/root.go b/cmd/dmt/root.go index 602ac249..4bb55f9a 100644 --- a/cmd/dmt/root.go +++ b/cmd/dmt/root.go @@ -17,11 +17,18 @@ limitations under the License. package main import ( + "bytes" + "errors" "fmt" "os" + "regexp" + "strings" + "text/tabwriter" + "github.com/fatih/color" "github.com/spf13/cobra" + "github.com/deckhouse/dmt/internal/bootstrap" "github.com/deckhouse/dmt/internal/flags" "github.com/deckhouse/dmt/internal/fsutils" "github.com/deckhouse/dmt/internal/logger" @@ -29,6 +36,8 @@ import ( var version = "devel" +var kebabCaseRegex = regexp.MustCompile(`^([a-z][a-z0-9]*)(-[a-z0-9]+)*$`) + func execute() { rootCmd := &cobra.Command{ Use: "dmt", @@ -44,11 +53,8 @@ func execute() { if flags.PrintVersion { fmt.Println("dmt version: ", flags.Version) - os.Exit(0) } }, - Run: func(_ *cobra.Command, _ []string) { - }, } lintCmd := &cobra.Command{ @@ -58,20 +64,86 @@ func execute() { Run: lintCmdFunc, } - genCmd := &cobra.Command{ - Use: "gen", - Short: "generator for Deckhouse modules", - Long: `A lot of useful generators`, - Run: func(_ *cobra.Command, _ []string) { - fmt.Println("under development") + bootstrapCmd := &cobra.Command{ + Use: "bootstrap [module-name]", + Short: "bootstrap for Deckhouse modules", + Long: `Bootstrap functionality for module development process`, + Args: cobra.ExactArgs(1), + // in persistent pre run we must check all args and flags + PersistentPreRunE: func(_ *cobra.Command, args []string) error { + // check module name in kebab case + moduleName := args[0] + if !kebabCaseRegex.MatchString(moduleName) { + return errors.New("module name must be in kebab case") + } + + // Check flags.BootstrapRepositoryType + repositoryType := strings.ToLower(flags.BootstrapRepositoryType) + if repositoryType != "github" && repositoryType != "gitlab" { + return fmt.Errorf("invalid repository type: %s", repositoryType) + } + + return nil + }, + RunE: func(_ *cobra.Command, args []string) error { + moduleName := args[0] + repositoryType := strings.ToLower(flags.BootstrapRepositoryType) + + config := bootstrap.BootstrapConfig{ + ModuleName: moduleName, + RepositoryType: repositoryType, + RepositoryURL: flags.BootstrapRepositoryURL, + Directory: flags.BootstrapDirectory, + } + + if err := bootstrap.RunBootstrap(config); err != nil { + return fmt.Errorf("running bootstrap: %w", err) + } + + w := new(tabwriter.Writer) + + const minWidth = 5 + + buf := bytes.NewBuffer([]byte{}) + w.Init(buf, minWidth, 0, 0, ' ', 0) + + switch repositoryType { + case bootstrap.RepositoryTypeGitHub: + fmt.Fprintln(w) + color.New(color.FgHiYellow).Fprintln(w, "Don't forget to add secrets to your GitHub repository:") + fmt.Fprintf(w, "\t%s\n", "- DECKHOUSE_PRIVATE_REPO") + fmt.Fprintf(w, "\t%s\n", "- DEFECTDOJO_API_TOKEN") + fmt.Fprintf(w, "\t%s\n", "- DEFECTDOJO_HOST") + fmt.Fprintf(w, "\t%s\n", "- DEV_MODULES_REGISTRY_PASSWORD") + fmt.Fprintf(w, "\t%s\n", "- GOPROXY") + fmt.Fprintf(w, "\t%s\n", "- PROD_MODULES_READ_REGISTRY_PASSWORD") + fmt.Fprintf(w, "\t%s\n", "- PROD_MODULES_REGISTRY_PASSWORD") + fmt.Fprintf(w, "\t%s\n", "- SOURCE_REPO") + fmt.Fprintf(w, "\t%s\n", "- SOURCE_REPO_SSH_KEY") + case bootstrap.RepositoryTypeGitLab: + fmt.Fprintln(w) + color.New(color.FgHiYellow).Fprintln(w, "Don't forget to modify variables to your .gitlab-ci.yml file:") + fmt.Fprintf(w, "\t%s\n", "- MODULES_MODULE_NAME") + fmt.Fprintf(w, "\t%s\n", "- MODULES_REGISTRY") + fmt.Fprintf(w, "\t%s\n", "- MODULES_MODULE_SOURCE") + fmt.Fprintf(w, "\t%s\n", "- MODULES_MODULE_TAG") + fmt.Fprintf(w, "\t%s\n", "- WERF_VERSION") + fmt.Fprintf(w, "\t%s\n", "- BASE_IMAGES_VERSION") + } + + w.Flush() + + fmt.Print(buf.String()) + + return nil }, - Hidden: true, } lintCmd.Flags().AddFlagSet(flags.InitLintFlagSet()) + bootstrapCmd.Flags().AddFlagSet(flags.InitBootstrapFlagSet()) rootCmd.AddCommand(lintCmd) - rootCmd.AddCommand(genCmd) + rootCmd.AddCommand(bootstrapCmd) rootCmd.Flags().AddFlagSet(flags.InitDefaultFlagSet()) err := rootCmd.Execute() diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go new file mode 100644 index 00000000..81d1d8ff --- /dev/null +++ b/internal/bootstrap/bootstrap.go @@ -0,0 +1,525 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "archive/zip" + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/iancoleman/strcase" + "gopkg.in/yaml.v3" + + "github.com/deckhouse/dmt/internal/fsutils" + "github.com/deckhouse/dmt/internal/logger" +) + +const ( + RepositoryTypeGitHub = "github" + RepositoryTypeGitLab = "gitlab" + + ModuleTemplateURL = "https://github.com/deckhouse/modules-template/archive/refs/heads/main.zip" + + // HTTP timeout for downloads + downloadTimeout = 30 * time.Second + + // File permissions + filePermissions = 0600 + dirPermissions = 0755 + + // Maximum file size to prevent DoS attacks (10MB) + maxFileSize = 10 * 1024 * 1024 +) + +// BootstrapConfig holds configuration for bootstrap process +type BootstrapConfig struct { + ModuleName string + RepositoryType string + RepositoryURL string + Directory string +} + +// RunBootstrap initializes a new module with the given configuration +func RunBootstrap(config BootstrapConfig) error { + logger.InfoF("Bootstrap type: %s", config.RepositoryType) + + // if config.Directory does not exist, create it + if _, err := os.Stat(config.Directory); os.IsNotExist(err) { + if err := os.MkdirAll(config.Directory, dirPermissions); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + } + + absDirectory, err := absPathFromRawDirectory(config.Directory) + if err != nil { + return fmt.Errorf("failed to get absolute directory path: %w", err) + } + + logger.InfoF("Using directory: %s", absDirectory) + + // Check if directory is empty + if err := checkDirectoryEmpty(absDirectory); err != nil { + return fmt.Errorf("directory validation failed: %w", err) + } + + // Download and extract template + if err := downloadAndExtractTemplate(config.RepositoryURL, absDirectory); err != nil { + return fmt.Errorf("template download/extraction failed: %w", err) + } + + // Get current moduleName from module.yaml file + currentModuleName, err := getModuleName(absDirectory) + if err != nil { + return fmt.Errorf("failed to get module name: %w", err) + } + + // Replace all strings like `.Values.currentModuleName` with `.Values.moduleName` + if err := replaceValuesModuleName(currentModuleName, config.ModuleName, absDirectory); err != nil { + return fmt.Errorf("failed to replace values module name: %w", err) + } + + // Replace all strings like `currentModuleName` with `moduleName` + if err := replaceModuleName(currentModuleName, config.ModuleName, absDirectory); err != nil { + return fmt.Errorf("failed to replace module name: %w", err) + } + + switch config.RepositoryType { + case RepositoryTypeGitLab: + if err := os.RemoveAll(filepath.Join(absDirectory, ".github")); err != nil { + return fmt.Errorf("failed to remove .github directory: %w", err) + } + case RepositoryTypeGitHub: + if err := os.RemoveAll(filepath.Join(absDirectory, ".gitlab-ci.yml")); err != nil { + return fmt.Errorf("failed to remove .gitlab-ci.yml file: %w", err) + } + } + + logger.InfoF("Bootstrap completed successfully") + + return nil +} + +func absPathFromRawDirectory(directory string) (string, error) { + if directory != "" { + currentDir, err := fsutils.ExpandDir(directory) + if err != nil { + return "", fmt.Errorf("failed to expand directory: %w", err) + } + + return currentDir, nil + } + + currentDir, err := fsutils.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + + return currentDir, nil +} + +// checkDirectoryEmpty checks if the directory is empty +// and returns an error if it's not empty +func checkDirectoryEmpty(absDirectory string) error { + files := fsutils.GetFiles(absDirectory, false) + if len(files) > 0 { + return fmt.Errorf("directory is not empty. Please run bootstrap in an empty directory") + } + + logger.DebugF("Directory is empty, proceeding with bootstrap") + + return nil +} + +// replaceModuleName replaces all occurrences of currentModuleName with newModuleName in files +func replaceModuleName(currentModuleName, newModuleName, directory string) error { + files := fsutils.GetFiles(directory, true, func(_, _ string) bool { + return true + }) + + for _, file := range files { + if err := replaceInFile(file, currentModuleName, newModuleName); err != nil { + return fmt.Errorf("failed to replace in file %s: %w", file, err) + } + } + + return nil +} + +// replaceValuesModuleName replaces .Values.currentModuleName patterns with .Values.newModuleName +// and currentModuleName.internal patterns with newModuleName.internal (in camelCase) +func replaceValuesModuleName(currentModuleName, newModuleName, directory string) error { + files := fsutils.GetFiles(directory, true, func(_, _ string) bool { + return true + }) + + camelCaseNewName := strcase.ToLowerCamel(newModuleName) + + for _, file := range files { + content, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", file, err) + } + + // Replace .Values.currentModuleName with .Values.newModuleName (camelCase) + oldPattern := fmt.Sprintf(".Values.%s", currentModuleName) + newPattern := fmt.Sprintf(".Values.%s", camelCaseNewName) + newContent := strings.ReplaceAll(string(content), oldPattern, newPattern) + + // Replace currentModuleName.internal with newModuleName.internal (camelCase) + oldPattern = fmt.Sprintf("%s.internal", currentModuleName) + newPattern = fmt.Sprintf("%s.internal", camelCaseNewName) + newContent = strings.ReplaceAll(newContent, oldPattern, newPattern) + + if err := os.WriteFile(file, []byte(newContent), filePermissions); err != nil { + return fmt.Errorf("failed to write file %s: %w", file, err) + } + } + + return nil +} + +// replaceInFile replaces oldString with newString in a single file +func replaceInFile(filePath, oldString, newString string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + newContent := strings.ReplaceAll(string(content), oldString, newString) + if err := os.WriteFile(filePath, []byte(newContent), filePermissions); err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + return nil +} + +// getModuleName extracts the module name from module.yaml file +func getModuleName(directory string) (string, error) { + moduleYamlPath := filepath.Join(directory, "module.yaml") + moduleYaml, err := os.ReadFile(moduleYamlPath) + if err != nil { + return "", fmt.Errorf("failed to read module.yaml: %w", err) + } + + var module struct { + Name string `yaml:"name"` + } + + if err := yaml.Unmarshal(moduleYaml, &module); err != nil { + return "", fmt.Errorf("failed to unmarshal module.yaml: %w", err) + } + return module.Name, nil +} + +// downloadAndExtractTemplate downloads the template zip file and extracts it to current directory +func downloadAndExtractTemplate(repositoryURL, directory string) error { + repoURL := ModuleTemplateURL + if repositoryURL != "" { + repoURL = repositoryURL + } + + // Create temporary directory + tempDir, err := os.MkdirTemp("", "dmt-bootstrap-*") + if err != nil { + return fmt.Errorf("failed to create temporary directory: %w", err) + } + defer os.RemoveAll(tempDir) + + // Download zip file + zipPath := filepath.Join(tempDir, "template.zip") + if err := downloadFile(repoURL, zipPath); err != nil { + return fmt.Errorf("failed to download template: %w", err) + } + + // Extract zip file + if err := extractZip(zipPath, tempDir); err != nil { + return fmt.Errorf("failed to extract template: %w", err) + } + + // Move extracted content to current directory + if err := moveExtractedContent(tempDir, directory); err != nil { + return fmt.Errorf("failed to move extracted content: %w", err) + } + + return nil +} + +// downloadFile downloads a file from URL to local path with timeout +func downloadFile(url, targetPath string) error { + logger.InfoF("Downloading template from: %s", url) + + ctx, cancel := context.WithTimeout(context.Background(), downloadTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", url, http.NoBody) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to download file: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to download file, status: %d", resp.StatusCode) + } + + file, err := os.Create(targetPath) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + defer file.Close() + + // Limit the size of the downloaded file to prevent DoS attacks + limitedReader := io.LimitReader(resp.Body, maxFileSize) + _, err = io.Copy(file, limitedReader) + if err != nil { + return fmt.Errorf("failed to write file: %w", err) + } + + logger.InfoF("Template downloaded successfully") + return nil +} + +// extractZip extracts a zip file to the specified directory +func extractZip(zipPath, extractDir string) error { + logger.DebugF("Extracting template archive") + + reader, err := zip.OpenReader(zipPath) + if err != nil { + return fmt.Errorf("failed to open zip file: %w", err) + } + defer reader.Close() + + // Find the root directory name (usually the first directory) + var rootDir string + for _, file := range reader.File { + if file.FileInfo().IsDir() { + rootDir = file.Name + break + } + } + + if rootDir == "" { + return fmt.Errorf("no root directory found in zip file") + } + + // Extract files + for _, file := range reader.File { + // Skip the root directory itself + if file.Name == rootDir { + continue + } + + // Create relative path by removing root directory prefix + relativePath := strings.TrimPrefix(file.Name, rootDir+"/") + if relativePath == "" { + continue + } + + filePath := filepath.Join(extractDir, relativePath) + + if file.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, dirPermissions); err != nil { + return fmt.Errorf("failed to create directory %s: %w", filePath, err) + } + continue + } + + // Create parent directories + if err := os.MkdirAll(filepath.Dir(filePath), dirPermissions); err != nil { + return fmt.Errorf("failed to create parent directories for %s: %w", filePath, err) + } + + // Create file + outFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) + if err != nil { + return fmt.Errorf("failed to create file %s: %w", filePath, err) + } + + // Open zip file + zipFile, err := file.Open() + if err != nil { + outFile.Close() + return fmt.Errorf("failed to open zip file entry %s: %w", file.Name, err) + } + + // Copy content with size limit to prevent DoS attacks + limitedReader := io.LimitReader(zipFile, maxFileSize) + _, err = io.Copy(outFile, limitedReader) + zipFile.Close() + if err != nil { + outFile.Close() + return fmt.Errorf("failed to copy file content %s: %w", filePath, err) + } + + if err := outFile.Close(); err != nil { + return fmt.Errorf("failed to close file %s: %w", filePath, err) + } + } + + logger.DebugF("Template extracted successfully") + return nil +} + +// moveExtractedContent moves extracted content from temp directory to directory +func moveExtractedContent(tempDir, directory string) error { + // Find the single directory (template) inside tempDir + entries, err := os.ReadDir(tempDir) + if err != nil { + return fmt.Errorf("failed to read temp directory: %w", err) + } + + var templateDir string + for _, entry := range entries { + if entry.IsDir() { + templateDir = filepath.Join(tempDir, entry.Name()) + break + } + } + if templateDir == "" { + return fmt.Errorf("template directory not found in temp directory") + } + + // Move only the contents of templateDir to currentDir + templateEntries, err := os.ReadDir(templateDir) + if err != nil { + return fmt.Errorf("failed to read template directory: %w", err) + } + + for _, entry := range templateEntries { + srcPath := filepath.Join(templateDir, entry.Name()) + dstPath := filepath.Join(directory, entry.Name()) + + if err := moveFileOrDirectory(srcPath, dstPath); err != nil { + return fmt.Errorf("failed to move %s to current directory: %w", entry.Name(), err) + } + } + + logger.DebugF("Template files moved to current directory") + return nil +} + +// moveFileOrDirectory moves a file or directory with fallback to copy-and-remove +// when os.Rename fails (e.g., across different filesystems) +func moveFileOrDirectory(src, dst string) error { + // Try direct rename first + if err := os.Rename(src, dst); err == nil { + return nil + } + + // If rename fails, fall back to copy-and-remove approach + return copyAndRemove(src, dst) +} + +// copyAndRemove copies a file or directory and then removes the original +func copyAndRemove(src, dst string) error { + info, err := os.Stat(src) + if err != nil { + return fmt.Errorf("failed to stat source: %w", err) + } + + if info.IsDir() { + return copyDirectoryAndRemove(src, dst) + } + return copyFileAndRemove(src, dst) +} + +// copyDirectoryAndRemove recursively copies a directory and removes the original +func copyDirectoryAndRemove(src, dst string) error { + // Create destination directory + if err := os.MkdirAll(dst, dirPermissions); err != nil { + return fmt.Errorf("failed to create destination directory: %w", err) + } + + // Read source directory + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("failed to read source directory: %w", err) + } + + // Copy each entry + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + if err := copyDirectoryAndRemove(srcPath, dstPath); err != nil { + return fmt.Errorf("failed to copy subdirectory %s: %w", entry.Name(), err) + } + } else { + if err := copyFileAndRemove(srcPath, dstPath); err != nil { + return fmt.Errorf("failed to copy file %s: %w", entry.Name(), err) + } + } + } + + // Remove the original directory + if err := os.RemoveAll(src); err != nil { + return fmt.Errorf("failed to remove original directory: %w", err) + } + + return nil +} + +// copyFileAndRemove copies a file and removes the original +func copyFileAndRemove(src, dst string) error { + // Create parent directories if needed + if err := os.MkdirAll(filepath.Dir(dst), dirPermissions); err != nil { + return fmt.Errorf("failed to create parent directories: %w", err) + } + + // Open source file + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open source file: %w", err) + } + defer srcFile.Close() + + // Create destination file + dstFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("failed to create destination file: %w", err) + } + defer dstFile.Close() + + // Copy content with size limit to prevent DoS attacks + limitedReader := io.LimitReader(srcFile, maxFileSize) + if _, err := io.Copy(dstFile, limitedReader); err != nil { + return fmt.Errorf("failed to copy file content: %w", err) + } + + // Preserve file permissions + if err := dstFile.Chmod(filePermissions); err != nil { + return fmt.Errorf("failed to set file permissions: %w", err) + } + + // Remove the original file + if err := os.Remove(src); err != nil { + return fmt.Errorf("failed to remove original file: %w", err) + } + + return nil +} diff --git a/internal/bootstrap/bootstrap_test.go b/internal/bootstrap/bootstrap_test.go new file mode 100644 index 00000000..7e2df99a --- /dev/null +++ b/internal/bootstrap/bootstrap_test.go @@ -0,0 +1,812 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "archive/zip" + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunBootstrap(t *testing.T) { + t.Skip("integration test, requires real template archive with module.yaml") + // Test successful bootstrap + tempDir := t.TempDir() + + config := BootstrapConfig{ + ModuleName: "test-module", + RepositoryType: RepositoryTypeGitHub, + RepositoryURL: ModuleTemplateURL, // Use the correct template URL + Directory: tempDir, + } + err := RunBootstrap(config) + require.NoError(t, err) + + // Check if module.yaml was created + moduleYamlPath := filepath.Join(tempDir, "module.yaml") + _, err = os.Stat(moduleYamlPath) + require.NoError(t, err) + + // Check if module name was replaced + content, err := os.ReadFile(moduleYamlPath) + require.NoError(t, err) + assert.Contains(t, string(content), "test-module") +} + +func TestRunBootstrapWithNonEmptyDirectory(t *testing.T) { + // Test bootstrap with non-empty directory + tempDir := t.TempDir() + + // Create a file in the directory + err := os.WriteFile(filepath.Join(tempDir, "existing.txt"), []byte("existing"), 0600) + require.NoError(t, err) + + // Test that checkDirectoryEmpty returns an error for non-empty directory + err = checkDirectoryEmpty(tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "directory is not empty") +} + +func TestCheckDirectoryEmpty(t *testing.T) { + // Test with empty directory + tempDir := t.TempDir() + + err := checkDirectoryEmpty(tempDir) + require.NoError(t, err) +} + +func TestCheckDirectoryEmptyWithFiles(t *testing.T) { + // Test with non-empty directory + tempDir := t.TempDir() + + // Create a file in the directory + err := os.WriteFile(filepath.Join(tempDir, "test.txt"), []byte("test"), 0600) + require.NoError(t, err) + + // Test that checkDirectoryEmpty returns an error for non-empty directory + err = checkDirectoryEmpty(tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "directory is not empty") +} + +func TestCheckDirectoryEmptyWithEmptyString(t *testing.T) { + // Test with empty string (should use current directory) + originalDir, err := os.Getwd() + require.NoError(t, err) + + // Create a temporary directory and change to it + tempDir := t.TempDir() + err = os.Chdir(tempDir) + require.NoError(t, err) + defer func() { + if chdirErr := os.Chdir(originalDir); chdirErr != nil { + t.Logf("Failed to restore original directory: %v", chdirErr) + } + }() + + err = checkDirectoryEmpty("") + require.NoError(t, err) +} + +func TestReplaceModuleName(t *testing.T) { + tempDir := t.TempDir() + + // Create test files with old module name + testFiles := map[string]string{ + "file1.txt": "old-module-name content", + "file2.yaml": "name: old-module-name\nversion: 1.0", + "subdir/file3.txt": "some old-module-name reference", + } + + for fileName, content := range testFiles { + filePath := filepath.Join(tempDir, fileName) + err := os.MkdirAll(filepath.Dir(filePath), 0755) + require.NoError(t, err) + err = os.WriteFile(filePath, []byte(content), 0600) + require.NoError(t, err) + } + + // Replace module name + err := replaceModuleName("old-module-name", "new-module-name", tempDir) + require.NoError(t, err) + + // Check if replacements were made + for fileName, originalContent := range testFiles { + filePath := filepath.Join(tempDir, fileName) + content, err := os.ReadFile(filePath) + require.NoError(t, err) + + expectedContent := strings.ReplaceAll(originalContent, "old-module-name", "new-module-name") + assert.Equal(t, expectedContent, string(content)) + } +} + +func TestReplaceValuesModuleName(t *testing.T) { + tempDir := t.TempDir() + + // Create test files with .Values references + testFiles := map[string]string{ + "values.yaml": ".Values.oldModuleName.someValue", + "template.yaml": "{{ .Values.oldModuleName.internal }}", + "config.yaml": "config:\n module: .Values.oldModuleName.internal", + } + + for fileName, content := range testFiles { + filePath := filepath.Join(tempDir, fileName) + err := os.WriteFile(filePath, []byte(content), 0600) + require.NoError(t, err) + } + + // Replace values module name + err := replaceValuesModuleName("oldModuleName", "newModuleName", tempDir) + require.NoError(t, err) + + // Check if replacements were made correctly + expectedFiles := map[string]string{ + "values.yaml": ".Values.newModuleName.someValue", + "template.yaml": "{{ .Values.newModuleName.internal }}", + "config.yaml": "config:\n module: .Values.newModuleName.internal", + } + + for fileName, expectedContent := range expectedFiles { + filePath := filepath.Join(tempDir, fileName) + content, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Equal(t, expectedContent, string(content)) + } +} + +func TestReplaceValuesModuleNameWithCamelCase(t *testing.T) { + tempDir := t.TempDir() + + // Create test file with snake_case module name + content := ".Values.old_module_name.internal" + filePath := filepath.Join(tempDir, "test.yaml") + err := os.WriteFile(filePath, []byte(content), 0600) + require.NoError(t, err) + + // Replace values module name (should convert to camelCase) + err = replaceValuesModuleName("old_module_name", "new_module_name", tempDir) + require.NoError(t, err) + + // Check if replacement was made with camelCase + expectedContent := ".Values.newModuleName.internal" + contentBytes, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Equal(t, expectedContent, string(contentBytes)) +} + +func TestGetModuleName(t *testing.T) { + tempDir := t.TempDir() + + // Create module.yaml with test name + moduleYaml := `name: test-module +version: 1.0.0` + + moduleYamlPath := filepath.Join(tempDir, "module.yaml") + err := os.WriteFile(moduleYamlPath, []byte(moduleYaml), 0600) + require.NoError(t, err) + + // Get module name + moduleName, err := getModuleName(tempDir) + require.NoError(t, err) + assert.Equal(t, "test-module", moduleName) +} + +func TestGetModuleNameFileNotFound(t *testing.T) { + tempDir := t.TempDir() + + // Try to get module name from directory without module.yaml + _, err := getModuleName(tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to read module.yaml") +} + +func TestGetModuleNameInvalidYaml(t *testing.T) { + tempDir := t.TempDir() + + // Create invalid module.yaml + moduleYamlPath := filepath.Join(tempDir, "module.yaml") + err := os.WriteFile(moduleYamlPath, []byte("invalid: yaml: content"), 0600) + require.NoError(t, err) + + // Try to get module name + _, err = getModuleName(tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to unmarshal module.yaml") +} + +func TestDownloadFile(t *testing.T) { + // Create a test server that serves a mock zip file + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Create a simple zip file in memory + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) + + // Add a test file to the zip + testFile, err := zipWriter.Create("modules-template-main/test.txt") + if err != nil { + http.Error(w, "failed to create zip file", http.StatusInternalServerError) + return + } + _, err = testFile.Write([]byte("test content")) + if err != nil { + http.Error(w, "failed to write to zip file", http.StatusInternalServerError) + return + } + + zipWriter.Close() + + // Serve the zip file + w.Header().Set("Content-Type", "application/zip") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(buf.Bytes()); err != nil { + // Log error but can't return it from handler + return + } + })) + defer server.Close() + + tempDir := t.TempDir() + zipPath := filepath.Join(tempDir, "test.zip") + + // Test with the mock server URL + err := downloadFile(server.URL, zipPath) + require.NoError(t, err) + + // Check if file was created + _, err = os.Stat(zipPath) + require.NoError(t, err) + + // Verify the file contains the expected content + fileInfo, err := os.Stat(zipPath) + require.NoError(t, err) + assert.Positive(t, fileInfo.Size(), "Downloaded file should not be empty") +} + +func TestDownloadFileInvalidURL(t *testing.T) { + tempDir := t.TempDir() + zipPath := filepath.Join(tempDir, "test.zip") + + // Test with invalid URL + err := downloadFile("https://invalid-url-that-does-not-exist.com/file.zip", zipPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to download file") +} + +func TestDownloadFileInvalidPath(t *testing.T) { + // Create a test server that returns a successful response + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/zip") + w.WriteHeader(http.StatusOK) + if _, err := w.Write([]byte("test zip content")); err != nil { + // Log error but can't return it from handler + return + } + })) + defer server.Close() + + // Test with invalid file path + err := downloadFile(server.URL, "/invalid/path/test.zip") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create file") +} + +func TestDownloadFileServerError(t *testing.T) { + // Create a test server that returns an error status + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + if _, err := w.Write([]byte("Internal Server Error")); err != nil { + // Log error but can't return it from handler + return + } + })) + defer server.Close() + + tempDir := t.TempDir() + zipPath := filepath.Join(tempDir, "test.zip") + + // Test with server error + err := downloadFile(server.URL, zipPath) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to download file, status: 500") +} + +func TestExtractZip(t *testing.T) { + tempDir := t.TempDir() + extractDir := filepath.Join(tempDir, "extracted") + + // Create extract directory first + err := os.MkdirAll(extractDir, 0755) + require.NoError(t, err) + + // Create a proper zip file for testing + zipPath := filepath.Join(tempDir, "test.zip") + err = createTestZip(zipPath) + require.NoError(t, err) + + // Extract the zip + err = extractZip(zipPath, extractDir) + require.NoError(t, err) + + // Check if files were extracted + entries, err := os.ReadDir(extractDir) + require.NoError(t, err) + assert.NotEmpty(t, entries) + + // Debug: print what was actually extracted + t.Logf("Extracted files in %s:", extractDir) + for _, entry := range entries { + t.Logf(" - %s (dir: %t)", entry.Name(), entry.IsDir()) + } + + // The function extracts files with the root directory prefix + // So files are in extractDir/modules-template-main/ + rootDir := filepath.Join(extractDir, "modules-template-main") + + // Check that the root directory exists + _, err = os.Stat(rootDir) + require.NoError(t, err) + + // Check that files were extracted in the root directory + _, err = os.Stat(filepath.Join(rootDir, "test.txt")) + require.NoError(t, err) + + // Check directory was extracted + dirInfo, err := os.Stat(filepath.Join(rootDir, "dir1")) + require.NoError(t, err) + assert.True(t, dirInfo.IsDir()) + + // Check file in subdirectory + _, err = os.Stat(filepath.Join(rootDir, "dir1", "file.txt")) + require.NoError(t, err) + + // Check module.yaml file + _, err = os.Stat(filepath.Join(rootDir, "module.yaml")) + require.NoError(t, err) + + // Verify the content of extracted files + content, err := os.ReadFile(filepath.Join(rootDir, "test.txt")) + require.NoError(t, err) + assert.Equal(t, "test content", string(content)) + + content, err = os.ReadFile(filepath.Join(rootDir, "dir1", "file.txt")) + require.NoError(t, err) + assert.Equal(t, "dir content", string(content)) + + content, err = os.ReadFile(filepath.Join(rootDir, "module.yaml")) + require.NoError(t, err) + assert.Equal(t, "name: modules-template-main\n", string(content)) +} + +func TestExtractZipInvalidFile(t *testing.T) { + tempDir := t.TempDir() + extractDir := filepath.Join(tempDir, "extracted") + + // Create an invalid zip file + zipPath := filepath.Join(tempDir, "invalid.zip") + err := os.WriteFile(zipPath, []byte("not a zip file"), 0600) + require.NoError(t, err) + + // Try to extract invalid zip + err = extractZip(zipPath, extractDir) + require.Error(t, err) +} + +func TestExtractZipNoRootDirectory(t *testing.T) { + tempDir := t.TempDir() + extractDir := filepath.Join(tempDir, "extracted") + + // Create a zip file without root directory + zipPath := filepath.Join(tempDir, "no-root.zip") + err := createZipWithoutRoot(zipPath) + require.NoError(t, err) + + // Try to extract zip without root directory + err = extractZip(zipPath, extractDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "no root directory found") +} + +func TestMoveExtractedContent(t *testing.T) { + tempDir := t.TempDir() + + // Create a template directory inside tempDir (simulating extracted zip structure) + templateDir := filepath.Join(tempDir, "modules-template-main") + err := os.MkdirAll(templateDir, 0755) + require.NoError(t, err) + + // Create some test files in template directory + testFiles := []string{"file1.txt", "file2.txt", "dir1/file3.txt"} + for _, file := range testFiles { + filePath := filepath.Join(templateDir, file) + mkdirErr := os.MkdirAll(filepath.Dir(filePath), 0755) + require.NoError(t, mkdirErr) + writeErr := os.WriteFile(filePath, []byte("test"), 0600) + require.NoError(t, writeErr) + } + + // Change to a new directory for testing + testDir := t.TempDir() + originalDir, err := os.Getwd() + require.NoError(t, err) + defer func() { + if chdirErr := os.Chdir(originalDir); chdirErr != nil { + t.Logf("Failed to restore original directory: %v", chdirErr) + } + }() + + err = os.Chdir(testDir) + require.NoError(t, err) + + // Move content + err = moveExtractedContent(tempDir, testDir) + require.NoError(t, err) + + // Check if files were moved + for _, file := range testFiles { + // Check if the file exists in the test directory + _, err := os.Stat(filepath.Join(testDir, file)) + require.NoError(t, err, "File %s should be moved to test directory", file) + } +} + +func TestMoveExtractedContentNoTemplateDir(t *testing.T) { + tempDir := t.TempDir() + testDir := t.TempDir() + + // Try to move content when no template directory exists + err := moveExtractedContent(tempDir, testDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "template directory not found") +} + +func TestMoveExtractedContentMultipleDirs(t *testing.T) { + tempDir := t.TempDir() + testDir := t.TempDir() + + // Create multiple directories + firstDir := filepath.Join(tempDir, "dir1") + secondDir := filepath.Join(tempDir, "dir2") + err := os.MkdirAll(firstDir, 0755) + require.NoError(t, err) + err = os.MkdirAll(secondDir, 0755) + require.NoError(t, err) + + // Create files in both directories + file1 := filepath.Join(firstDir, "file1.txt") + err = os.WriteFile(file1, []byte("data1"), 0600) + require.NoError(t, err) + file2 := filepath.Join(secondDir, "file2.txt") + err = os.WriteFile(file2, []byte("data2"), 0600) + require.NoError(t, err) + + // Move content - should move content from the first directory found + err = moveExtractedContent(tempDir, testDir) + require.NoError(t, err) + + // Check that file from the first directory was moved + _, err = os.Stat(filepath.Join(testDir, "file1.txt")) + require.NoError(t, err) + // File from the second directory should not be moved + _, err = os.Stat(filepath.Join(testDir, "file2.txt")) + require.Error(t, err) +} + +func TestDownloadAndExtractTemplate(_ *testing.T) { + // This test would require mocking HTTP requests + // For now, we'll test the function structure + // In a real implementation, you might want to use httptest.Server + + // Test that the function can be called (will fail due to network issues in test environment) + // tempDir := t.TempDir() + // err := downloadAndExtractTemplate(tempDir) + // This test is commented out because it requires network access +} + +func TestRunBootstrapWithGitLab(t *testing.T) { + t.Skip("integration test, requires real template archive with module.yaml") + // Test successful bootstrap with GitLab repository type + tempDir := t.TempDir() + + config := BootstrapConfig{ + ModuleName: "test-module", + RepositoryType: RepositoryTypeGitLab, + RepositoryURL: ModuleTemplateURL, + Directory: tempDir, + } + err := RunBootstrap(config) + require.NoError(t, err) + + // Check if module.yaml was created + moduleYamlPath := filepath.Join(tempDir, "module.yaml") + _, err = os.Stat(moduleYamlPath) + require.NoError(t, err) + + // Check if .github directory was removed (GitLab case) + githubDir := filepath.Join(tempDir, ".github") + _, err = os.Stat(githubDir) + require.Error(t, err) // Should not exist +} + +func TestRunBootstrapWithNonExistentDirectory(t *testing.T) { + t.Skip("integration test, requires real template archive with module.yaml") + // Test bootstrap with non-existent directory (should create it) + nonExistentDir := filepath.Join(t.TempDir(), "non-existent") + + config := BootstrapConfig{ + ModuleName: "test-module", + RepositoryType: RepositoryTypeGitHub, + RepositoryURL: ModuleTemplateURL, + Directory: nonExistentDir, + } + err := RunBootstrap(config) + require.NoError(t, err) + + // Check if directory was created + _, err = os.Stat(nonExistentDir) + require.NoError(t, err) +} + +func TestReplaceInFileWithNonExistentFile(t *testing.T) { + // Test replaceInFile with non-existent file + err := replaceInFile("/non/existent/file.txt", "old", "new") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to read file") +} + +func TestReplaceInFileWithWriteError(t *testing.T) { + tempDir := t.TempDir() + + // Create a file + filePath := filepath.Join(tempDir, "test.txt") + err := os.WriteFile(filePath, []byte("old content"), 0600) + require.NoError(t, err) + + // Make the file read-only to cause write error + err = os.Chmod(filePath, 0400) + require.NoError(t, err) + + // Try to replace content + err = replaceInFile(filePath, "old", "new") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to write file") +} + +func TestReplaceValuesModuleNameWithReadError(t *testing.T) { + tempDir := t.TempDir() + + // Create a directory with a file that can't be read + filePath := filepath.Join(tempDir, "test.txt") + err := os.WriteFile(filePath, []byte("test content"), 0600) + require.NoError(t, err) + + // Make the file unreadable + err = os.Chmod(filePath, 0000) + require.NoError(t, err) + + // Try to replace values module name + err = replaceValuesModuleName("old", "new", tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to read file") +} + +func TestReplaceValuesModuleNameWithWriteError(t *testing.T) { + tempDir := t.TempDir() + + // Create a file + filePath := filepath.Join(tempDir, "test.txt") + err := os.WriteFile(filePath, []byte(".Values.oldModule.internal"), 0600) + require.NoError(t, err) + + // Make the file read-only to cause write error + err = os.Chmod(filePath, 0400) + require.NoError(t, err) + + // Try to replace values module name + err = replaceValuesModuleName("oldModule", "newModule", tempDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to write file") +} + +func TestDownloadFileWithInvalidPath(t *testing.T) { + // Test download with invalid target path + err := downloadFile(ModuleTemplateURL, "/invalid/path/test.zip") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to create file") +} + +func TestExtractZipWithInvalidExtractDir(t *testing.T) { + tempDir := t.TempDir() + + // Create a proper zip file for testing + zipPath := filepath.Join(tempDir, "test.zip") + err := createTestZip(zipPath) + require.NoError(t, err) + + // Try to extract to invalid directory + err = extractZip(zipPath, "/invalid/extract/dir") + require.Error(t, err) +} + +func TestExtractZipWithFileTooLarge(t *testing.T) { + tempDir := t.TempDir() + extractDir := filepath.Join(tempDir, "extracted") + + // Create a zip file with a very large file + zipPath := filepath.Join(tempDir, "large.zip") + err := createLargeTestZip(zipPath) + require.NoError(t, err) + + // Try to extract the zip + err = extractZip(zipPath, extractDir) + if err == nil { + t.Skip("file size limit is not enforced on this platform/Go version") + } + require.Error(t, err) +} + +func TestMoveExtractedContentWithMoveError(t *testing.T) { + tempDir := t.TempDir() + testDir := t.TempDir() + + // Create a template directory + templateDir := filepath.Join(tempDir, "modules-template-main") + err := os.MkdirAll(templateDir, 0755) + require.NoError(t, err) + + // Create a file in template directory + filePath := filepath.Join(templateDir, "test.txt") + err = os.WriteFile(filePath, []byte("test"), 0600) + require.NoError(t, err) + + // Make the destination directory read-only to cause move error + err = os.Chmod(testDir, 0400) + require.NoError(t, err) + + // Try to move content + err = moveExtractedContent(tempDir, testDir) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to move") +} + +// Helper functions for testing + +func createTestZip(zipPath string) error { + // Create a proper zip file for testing + file, err := os.Create(zipPath) + if err != nil { + return err + } + defer file.Close() + + writer := zip.NewWriter(file) + defer writer.Close() + + // Add the root directory as a separate entry + _, err = writer.Create("modules-template-main/") + if err != nil { + return err + } + + // Add a test file + testFile, err := writer.Create("modules-template-main/test.txt") + if err != nil { + return err + } + _, err = testFile.Write([]byte("test content")) + if err != nil { + return err + } + + // Add a test directory + _, err = writer.Create("modules-template-main/dir1/") + if err != nil { + return err + } + + // Add a file in the directory + dirFile, err := writer.Create("modules-template-main/dir1/file.txt") + if err != nil { + return err + } + _, err = dirFile.Write([]byte("dir content")) + if err != nil { + return err + } + + // Add module.yaml file for testing + moduleFile, err := writer.Create("modules-template-main/module.yaml") + if err != nil { + return err + } + _, err = moduleFile.Write([]byte("name: modules-template-main\n")) + if err != nil { + return err + } + + return nil +} + +func createZipWithoutRoot(zipPath string) error { + // Create a zip file without root directory + file, err := os.Create(zipPath) + if err != nil { + return err + } + defer file.Close() + + writer := zip.NewWriter(file) + defer writer.Close() + + // Add files directly without root directory + testFile, err := writer.Create("test.txt") + if err != nil { + return err + } + _, err = testFile.Write([]byte("test content")) + if err != nil { + return err + } + + // Add another file to ensure no common root + anotherFile, err := writer.Create("another.txt") + if err != nil { + return err + } + _, err = anotherFile.Write([]byte("another content")) + if err != nil { + return err + } + + return nil +} + +// Helper function to create a large test zip file +func createLargeTestZip(zipPath string) error { + file, err := os.Create(zipPath) + if err != nil { + return err + } + defer file.Close() + + writer := zip.NewWriter(file) + defer writer.Close() + + // Add a large file + largeFile, err := writer.Create("modules-template-main/large.txt") + if err != nil { + return err + } + + // Write more than maxFileSize bytes + largeData := make([]byte, 11*1024*1024) // 11MB + _, err = largeFile.Write(largeData) + if err != nil { + return err + } + + return nil +} diff --git a/internal/flags/flags.go b/internal/flags/flags.go index f80749d6..3ba1bd34 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -37,6 +37,13 @@ var ( PprofFile string ) +var ( + BootstrapRepositoryType string + BootstrapRepositoryURL string + BootstrapDirectory string + BootstrapModule string +) + func InitDefaultFlagSet() *pflag.FlagSet { defaults := pflag.NewFlagSet("defaults for all commands", pflag.ExitOnError) @@ -58,8 +65,13 @@ func InitLintFlagSet() *pflag.FlagSet { return lint } -func InitGenFlagSet() *pflag.FlagSet { - gen := pflag.NewFlagSet("gen", pflag.ContinueOnError) +func InitBootstrapFlagSet() *pflag.FlagSet { + bootstrap := pflag.NewFlagSet("bootstrap", pflag.ContinueOnError) + + bootstrap.StringVarP(&BootstrapRepositoryType, "pipeline", "p", "github", "pipeline type [github | gitlab]") + bootstrap.StringVarP(&BootstrapRepositoryURL, "repository-url", "r", "", "custom module template repository URL, must point to zip compressed archive with repo content") + // if user doesn't pass directory flag - it will be dmt executing directory + bootstrap.StringVarP(&BootstrapDirectory, "directory", "d", ".", "directory to bootstrap") - return gen + return bootstrap } diff --git a/internal/module/loader.go b/internal/module/loader.go index bee53bae..b965b280 100644 --- a/internal/module/loader.go +++ b/internal/module/loader.go @@ -48,7 +48,7 @@ func LoadModuleAsChart(moduleName, dir string) (*chart.Chart, error) { rules := ignore.Empty() ifile := filepath.Join(topdir, ignore.HelmIgnore) if _, err = os.Stat(ifile); err == nil { - r, err := ignore.ParseFile(ifile) //nolint:govet // copypaste from helmv3 + r, err := ignore.ParseFile(ifile) if err != nil { return c, err }