diff --git a/README.md b/README.md index e959544c..839f287e 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,25 @@ http: To sync repositories, use `minima sync`. To search for new MU repositories, use `minima updates -s`. + To search and sync automatically all the new MU repositories: use `minima updates`. +To cleanup obsolete repositories, use `--cleanup` option. It should be ran whether with +sync or updates accordingly: +``` +./minima sync --cleanup +``` +or +``` +./minima updates --cleanup +``` +If a repository that is not specified in the minima.yaml config file will be identified, +you will be prompted to keep or delete it. If you want to automatically delete all the +repositories that are not specified in minima.yaml, run: +``` +./minima sync --cleanup --auto-approve +``` ## How to contribute diff --git a/cmd/sync.go b/cmd/sync.go index 54d6e4f4..94b3af57 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -2,12 +2,14 @@ package cmd import ( "fmt" + "io/ioutil" "log" "net/url" "os" "path/filepath" "strings" + "github.com/manifoldco/promptui" "github.com/spf13/cobra" "github.com/uyuni-project/minima/get" yaml "gopkg.in/yaml.v2" @@ -166,7 +168,9 @@ func syncersFromConfig(configString string) (result []*get.Syncer, err error) { } result = append(result, get.NewSyncer(*repoURL, archs, storage)) } - + if cleanup { + RemoveOldChannelsFromFileStorage(config) + } return } @@ -176,3 +180,66 @@ func init() { RootCmd.PersistentFlags().StringVarP(&archs, "arch", "a", "", "flag that specifies covered archs in the given repo") RootCmd.PersistentFlags().BoolVarP(&syncLegacyPackages, "legacypackages", "l", false, "flag that triggers mirroring of i586 pkgs in x86_64 repos") } + +func RemoveOldChannelsFromFileStorage (config Config) (err error) { + // DO CLEANUP - TO BE IMPLEMENTED + log.Println("searching for outdated repositories...") + //fmt.Printf("List of Repos from config: %s ---> %s\n", config.SCC.RepoNames, config.HTTP) + mappedRepos := make(map[string]bool) + var urlink *url.URL + for _, elem := range config.HTTP { + urlink, err = url.Parse(elem.URL) + if err != nil { + panic(err) + } + mappedRepos[filepath.Join(config.Storage.Path, urlink.Path)] = true + } + //fmt.Printf("MAPPED REPOS: %v\n", mappedRepos) + path := config.Storage.Path + muChannelList := make(map[string]bool) + filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + if info.Name() == "repodata" { + muChannelList[strings.Replace(path, "/repodata", "", 1)] = true + return nil + } + if files, err := ioutil.ReadDir(path); len(files) == 0 && path != config.Storage.Path { + if err != nil { + log.Fatal(err) + } + log.Printf("Removing unused empty folders: %s\n", path) + os.RemoveAll(path) + } + } + return nil + }) + //fmt.Printf("CHANNEL LIST: %v\n", muChannelList) + for ind, _ := range muChannelList { + if mappedRepos[ind] { + log.Printf("Repo %s is registered...\n", ind) + } else { + log.Printf("Repo %s is not registered in the yaml file...", ind) + if autoApprove { + log.Printf("Removing repo %s ...\n", ind) + os.RemoveAll(ind) + } else { + prompt := promptui.Select{ + Label: fmt.Sprintf("Delete repo: %s ??? [Yes/No]", ind), + Items: []string{"Yes", "No"}, + } + _, result, err := prompt.Run() + if err != nil { + log.Fatalf("Prompt failed %v\n", err) + } + if result == "Yes" { + log.Printf("Removing repo: %s ...\n", ind) + os.RemoveAll(ind) + } else { + log.Printf("Keeping repo: %s ...\n", ind) + } + } + } + } + log.Println("...done!") + return +} diff --git a/cmd/updates.go b/cmd/updates.go index 2050793b..dab8cf2b 100644 --- a/cmd/updates.go +++ b/cmd/updates.go @@ -50,6 +50,7 @@ to quickly create a Cobra application.`, justSearch bool thisMU string cleanup bool + autoApprove bool ) func init() { @@ -58,6 +59,7 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&justSearch, "search", "s", false, "flag that would trigger only looking for updates on OBS") RootCmd.PersistentFlags().StringVarP(&thisMU, "maintupdate", "m", "", "flag that consumes the name of an MU, like 'SUSE:Maintenance:Incident:ReleaseRequest'") RootCmd.PersistentFlags().BoolVarP(&cleanup, "cleanup", "k", false, "flag that triggers cleaning up the storage (from old MU channels)") + RootCmd.PersistentFlags().BoolVar(&autoApprove, "auto-approve", false, "triggers automatic deleting of repositories") } func muFindAndSync() { @@ -74,7 +76,7 @@ func muFindAndSync() { if err != nil { log.Fatalf("There is an error: %v", err) } - err = RemoveOldChannels(config, updateList) + err = RemoveOldChannels(config, updateList, autoApprove) if err != nil { log.Fatalf("There is an error: %v", err) } @@ -261,33 +263,61 @@ func GetUpdatesAndChannels(usr, passwd string, justsearch bool) (updlist []Updat return updlist, err } -func RemoveOldChannels(config Config, updates []Updates) (err error) { +func RemoveOldChannels(config Config, updates []Updates, deleteNow bool) (err error) { mappedUpdates := MakeAMap(updates) + fmt.Printf("Mapped Updates: %s\n", mappedUpdates) switch config.Storage.Type { case "file": var muChannelList []string - err = filepath.Walk(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/"), func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - muChannelList = append(muChannelList, path) - } - return nil - }) - if err != nil { - return + if _, err := os.Stat(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/")); os.IsNotExist(err) { + log.Printf("This path: \"%s\" does not exist! Check your mirror...\n") + } else { + err = filepath.Walk(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/"), func(path string, info os.FileInfo, err error) error { + if info.Name() == "repodata" { + muChannelList = append(muChannelList, path) + } + if info.IsDir() { + if files, err := ioutil.ReadDir(path); len(files) == 0 { + if err != nil { + log.Fatal(err) + } + log.Printf("Removing unused empty folders: %s\n", path) + os.RemoveAll(filepath.Join(config.Storage.Path, "ibs/SUSE:/Maintenance:/", path)) + } + } + return nil + }) } - //templ := regexp.MustCompile(`/\d{5,6}/`) + if err != nil { + return + } + //List of repositories that will be deleted: reposToDel + var reposToDel map[string]bool + fmt.Printf("MuchannelList: %v\n", muChannelList) for _, elem := range muChannelList { - if regexp.MustCompile(`/\d{5,6}/`).FindString(elem) != "" { - _, exists := mappedUpdates[strings.Replace(regexp.MustCompile(`/\d{5,6}/`).FindString(elem), "/", "", 10)] - if !exists { - log.Printf("removing: %s...\n", elem) - err = os.RemoveAll(elem) - if err != nil { - return + if regexp.MustCompile(`/\d{5,6}/`).FindString(elem) != "" { + _, exists := mappedUpdates[strings.Replace(regexp.MustCompile(`/\d{5,6}/`).FindString(elem), "/", "", 10)] + if !exists { + if deleteNow { + log.Printf("removing: %s...\n", elem) + err = os.RemoveAll(elem) + if err != nil { + return + } + } else { + reposToDel[elem] = true + } } } - } - } + } + log.Println("Mirror will have following changes:") + if reposToDel == nil { + log.Println("...there is nothing to delete!") + } + for index, _ := range reposToDel { + fmt.Printf("- %s", index) + } + case "s3": } return } @@ -295,11 +325,23 @@ func RemoveOldChannels(config Config, updates []Updates) (err error) { func MakeAMap(updates []Updates) (updatesMap map[string]bool){ updatesMap = make(map[string]bool) for _, elem := range updates { + if elem.IncidentNumber == "" { + elem.IncidentNumber = fmt.Sprintf("%s", elem.Repositories[0]) + } updatesMap[elem.IncidentNumber] = true } return } +func CheckIFMapHasKey(updatesMap map[string]bool, prefix, elem string) (yes bool) { + for key, _ := range updatesMap { + if strings.Contains(key, strings.Replace(elem, prefix, "", 1)) { + yes = true + } + } + return +} + type Updates struct { IncidentNumber string ReleaseRequest string diff --git a/go.mod b/go.mod index 21044274..13573559 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 // indirect github.com/jtolds/gls v4.2.1+incompatible // indirect + github.com/manifoldco/promptui v0.8.0 github.com/smartystreets/assertions v0.0.0-20170818220048-9c0ea8acbc1d // indirect github.com/smartystreets/goconvey v0.0.0-20170825221426-e5b2b7c91115 // indirect github.com/smartystreets/gunit v0.0.0-20170705041418-d235ada78c07 // indirect diff --git a/go.sum b/go.sum index 6a591f76..90f0695e 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,7 @@ github.com/aws/aws-sdk-go v0.0.0-20170914235556-ad2c5a3ad388 h1:Aa5a+YIfo9fdEe2f github.com/aws/aws-sdk-go v0.0.0-20170914235556-ad2c5a3ad388/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -155,12 +156,22 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= +github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo= +github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -300,6 +311,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=