Support additional unversioned types

github.com/tektoncd/pipeline has an additional type in https://github.com/tektoncd/pipeline/tree/main/pkg/apis/pipeline/pod which isn't versioned, and shouldn't be its own package section in the generated doc, but is referenced in versioned packages under `pkg/apis/pipeline`. This probably isn't the ideal way to have things set up, but it's what we've got. In order to use this tool to generate reference docs for Tekton Pipeline properly, we need to include the type in that package.

To enable that, this adds an additional doc comment, `+gencrdrefdocs:unversionedTypes`. When that is found in a package's `doc.go` and that package isn't already being included normally, its types will be made available for reference by the "real" API packages, with `unversioned` as their version.

I also fixed the existing `docCommentForceIncludes` - `pkg.DocComments` doesn't include the `// ` prefix, so searching for `// +gencrdrefdocs:force` would always fail. Therefore, I removed the `// ` prefix.

Signed-off-by: Andrew Bayer <[email protected]>

Author: abayer

main.go

@@ -38,7 +38,8 @@ var (
 )
 
 const (
-	docCommentForceIncludes = "// +gencrdrefdocs:force"
+	docCommentForceIncludes           = "+gencrdrefdocs:force"
+	docCommentIncludeUnversionedTypes = "+gencrdrefdocs:unversionedTypes"
 )
 
 type generatorConfig struct {
@@ -130,22 +131,31 @@ func main() {
 	}
 
 	klog.Infof("parsing go packages in directory %s", *flAPIDir)
-	pkgs, err := parseAPIPackages(*flAPIDir)
+	pkgs, unversionedPkgs, err := parseAPIPackages(*flAPIDir)
 	if err != nil {
 		klog.Fatal(err)
 	}
 	if len(pkgs) == 0 {
 		klog.Fatalf("no API packages found in %s", *flAPIDir)
 	}
 
-	apiPackages, err := combineAPIPackages(pkgs)
+	var unversionedPkgNames []string
+	for _, uvp := range unversionedPkgs {
+		unversionedPkgNames = append(unversionedPkgNames, uvp.Path)
+	}
+	apiPackages, err := combineAPIPackages(pkgs, unversionedPkgNames)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	unversionedAPIPackages, err := combineAPIPackages(unversionedPkgs, unversionedPkgNames)
 	if err != nil {
 		klog.Fatal(err)
 	}
 
 	mkOutput := func() (string, error) {
 		var b bytes.Buffer
-		err := render(&b, apiPackages, config)
+		err := render(&b, apiPackages, unversionedAPIPackages, config)
 		if err != nil {
 			return "", errors.Wrap(err, "failed to render the result")
 		}
@@ -200,16 +210,17 @@ func groupName(pkg *types.Package) string {
 	return ""
 }
 
-func parseAPIPackages(dir string) ([]*types.Package, error) {
+func parseAPIPackages(dir string) ([]*types.Package, []*types.Package, error) {
 	b := parser.New()
 	// the following will silently fail (turn on -v=4 to see logs)
 	if err := b.AddDirRecursive(*flAPIDir); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	scan, err := b.FindTypes()
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to parse pkgs and types")
+		return nil, nil, errors.Wrap(err, "failed to parse pkgs and types")
 	}
+	var unversionedPkgs []*types.Package
 	var pkgNames []string
 	for p := range scan {
 		pkg := scan[p]
@@ -223,7 +234,10 @@ func parseAPIPackages(dir string) ([]*types.Package, error) {
 			continue
 		}
 
-		if groupName(pkg) != "" && len(pkg.Types) > 0 || containsString(pkg.DocComments, docCommentForceIncludes) {
+		if len(pkg.Types) > 0 && containsString(pkg.DocComments, docCommentIncludeUnversionedTypes) {
+			klog.Infof("including package=%s as an additional unversioned include", p)
+			unversionedPkgs = append(unversionedPkgs, pkg)
+		} else if groupName(pkg) != "" && len(pkg.Types) > 0 || containsString(pkg.DocComments, docCommentForceIncludes) {
 			klog.V(3).Infof("package=%v has groupName and has types", p)
 			pkgNames = append(pkgNames, p)
 		}
@@ -234,7 +248,7 @@ func parseAPIPackages(dir string) ([]*types.Package, error) {
 		klog.Infof("using package=%s", p)
 		pkgs = append(pkgs, scan[p])
 	}
-	return pkgs, nil
+	return pkgs, unversionedPkgs, nil
 }
 
 func containsString(sl []string, str string) bool {
@@ -248,7 +262,7 @@ func containsString(sl []string, str string) bool {
 
 // combineAPIPackages groups the Go packages by the <apiGroup+apiVersion> they
 // offer, and combines the types in them.
-func combineAPIPackages(pkgs []*types.Package) ([]*apiPackage, error) {
+func combineAPIPackages(pkgs []*types.Package, unversionedPkgNames []string) ([]*apiPackage, error) {
 	pkgMap := make(map[string]*apiPackage)
 	var pkgIds []string
 
@@ -263,7 +277,7 @@ func combineAPIPackages(pkgs []*types.Package) ([]*apiPackage, error) {
 	}
 
 	for _, pkg := range pkgs {
-		apiGroup, apiVersion, err := apiVersionForPackage(pkg)
+		apiGroup, apiVersion, err := apiVersionForPackage(pkg, unversionedPkgNames)
 		if err != nil {
 			return nil, errors.Wrapf(err, "could not get apiVersion for package %s", pkg.Path)
 		}
@@ -597,8 +611,13 @@ func isOptionalMember(m types.Member) bool {
 	return ok
 }
 
-func apiVersionForPackage(pkg *types.Package) (string, string, error) {
+func apiVersionForPackage(pkg *types.Package, unversionedPkgNames []string) (string, string, error) {
 	group := groupName(pkg)
+	for _, upn := range unversionedPkgNames {
+		if upn == pkg.Path {
+			return group, "unversioned", nil
+		}
+	}
 	version := pkg.Name // assumes basename (i.e. "v1" in "core/v1") is apiVersion
 	r := `^v\d+((alpha|beta)[a-z0-9]+)?$`
 	if !regexp.MustCompile(r).MatchString(version) {
@@ -648,9 +667,9 @@ func constantsOfType(t *types.Type, pkg *apiPackage) []*types.Type {
 	return sortTypes(constants)
 }
 
-func render(w io.Writer, pkgs []*apiPackage, config generatorConfig) error {
-	references := findTypeReferences(pkgs)
-	typePkgMap := extractTypeToPackageMap(pkgs)
+func render(w io.Writer, pkgs []*apiPackage, unversionedPkgs []*apiPackage, config generatorConfig) error {
+	references := findTypeReferences(append(pkgs, unversionedPkgs...))
+	typePkgMap := extractTypeToPackageMap(append(pkgs, unversionedPkgs...))
 
 	t, err := template.New("").Funcs(map[string]interface{}{
 		"isExportedType":     isExportedType,