docs: api separation (#3630)

# What does this PR do?

First step towards cleaning up the API reference section of the docs.

- Separates API reference into 3 sections: stable (`v1`), experimental (`v1alpha` and `v1beta`), and deprecated (`deprecated=True`)
- Each section is accessible via the dropdown menu and `docs/api-overview`

<img width="1237" height="321" alt="Screenshot 2025-09-30 at 5 47 30 PM" src="https://github.com/user-attachments/assets/fe0e498c-b066-46ed-a48e-4739d3b6724c" />

<img width="860" height="510" alt="Screenshot 2025-09-30 at 5 47 49 PM" src="https://github.com/user-attachments/assets/a92a8d8c-94bf-42d5-9f5b-b47bb2b14f9c" />

- Deprecated APIs: Added styling to the sidebar, and a notice on the endpoint pages

<img width="867" height="428" alt="Screenshot 2025-09-30 at 5 47 43 PM" src="https://github.com/user-attachments/assets/9e6e050d-c782-461b-8084-5ff6496d7bd9" />

Closes #3628

TODO in follow-up PRs:

- Add the ability to annotate API groups with supplementary content  (so we can have longer descriptions of complex APIs like Responses)
- Clean up docstrings to show API endpoints (or short semantic titles) in the sidebar

## Test Plan

- Local testing
- Made sure API conformance test still passes

Author: reluctantfuturist

docs/docs/api-overview.md

@@ -0,0 +1,49 @@
+# API Reference Overview
+
+The Llama Stack provides a comprehensive set of APIs organized by stability level to help you choose the right endpoints for your use case.
+
+## 🟢 Stable APIs
+
+**Production-ready APIs with backward compatibility guarantees.**
+
+These APIs are fully tested, documented, and stable. They follow semantic versioning principles and maintain backward compatibility within major versions. Recommended for production applications.
+
+[**Browse Stable APIs →**](./api/llama-stack-specification)
+
+**Key Features:**
+- ✅ Backward compatibility guaranteed
+- ✅ Comprehensive testing and validation
+- ✅ Production-ready reliability
+- ✅ Long-term support
+
+---
+
+## 🟡 Experimental APIs
+
+**Preview APIs that may change before becoming stable.**
+
+These APIs include v1alpha and v1beta endpoints that are feature-complete but may undergo changes based on feedback. Great for exploring new capabilities and providing feedback.
+
+[**Browse Experimental APIs →**](./api-experimental/llama-stack-specification-experimental-apis)
+
+**Key Features:**
+- 🧪 Latest features and capabilities
+- 🧪 May change based on user feedback
+- 🧪 Active development and iteration
+- 🧪 Opportunity to influence final design
+
+---
+
+## 🔴 Deprecated APIs
+
+**Legacy APIs for migration reference.**
+
+These APIs are deprecated and will be removed in future versions. They are provided for migration purposes and to help transition to newer, stable alternatives.
+
+[**Browse Deprecated APIs →**](./api-deprecated/llama-stack-specification-deprecated-apis)
+
+**Key Features:**
+- ⚠️ Will be removed in future versions
+- ⚠️ Migration guidance provided
+- ⚠️ Use for compatibility during transition
+- ⚠️ Not recommended for new projects

docs/docs/providers/agents/index.mdx

@@ -1,12 +1,7 @@
 ---
-description: "Agents API for creating and interacting with agentic systems.
+description: "Agents
 
-    Main functionalities provided by this API:
-    - Create agents with specific instructions and ability to use tools.
-    - Interactions with agents are grouped into sessions (\"threads\"), and each interaction is called a \"turn\".
-    - Agents can be provided with various tools (see the ToolGroups and ToolRuntime APIs for more details).
-    - Agents can be provided with various shields (see the Safety API for more details).
-    - Agents can also use Memory to retrieve information from knowledge bases. See the RAG Tool and Vector IO APIs for more details."
+    APIs for creating and interacting with agentic systems."
 sidebar_label: Agents
 title: Agents
 ---
@@ -15,13 +10,8 @@ title: Agents
 
 ## Overview
 
-Agents API for creating and interacting with agentic systems.
+Agents
 
-    Main functionalities provided by this API:
-    - Create agents with specific instructions and ability to use tools.
-    - Interactions with agents are grouped into sessions ("threads"), and each interaction is called a "turn".
-    - Agents can be provided with various tools (see the ToolGroups and ToolRuntime APIs for more details).
-    - Agents can be provided with various shields (see the Safety API for more details).
-    - Agents can also use Memory to retrieve information from knowledge bases. See the RAG Tool and Vector IO APIs for more details.
+    APIs for creating and interacting with agentic systems.
 
 This section contains documentation for all available providers for the **agents** API.

docs/docusaurus.config.ts

@@ -55,10 +55,27 @@ const config: Config = {
           label: 'Docs',
         },
         {
-          type: 'docSidebar',
-          sidebarId: 'apiSidebar',
-          position: 'left',
+          type: 'dropdown',
           label: 'API Reference',
+          position: 'left',
+          to: '/docs/api-overview',
+          items: [
+            {
+              type: 'docSidebar',
+              sidebarId: 'stableApiSidebar',
+              label: '🟢 Stable APIs',
+            },
+            {
+              type: 'docSidebar',
+              sidebarId: 'experimentalApiSidebar',
+              label: '🟡 Experimental APIs',
+            },
+            {
+              type: 'docSidebar',
+              sidebarId: 'deprecatedApiSidebar',
+              label: '🔴 Deprecated APIs',
+            },
+          ],
         },
         {
           href: 'https://github.com/llamastack/llama-stack',
@@ -83,7 +100,7 @@ const config: Config = {
             },
             {
               label: 'API Reference',
-              to: '/docs/api/llama-stack-specification',
+              to: '/docs/api-overview',
             },
           ],
         },
@@ -170,7 +187,7 @@ const config: Config = {
         id: "openapi",
         docsPluginId: "classic",
         config: {
-          llamastack: {
+          stable: {
             specPath: "static/llama-stack-spec.yaml",
             outputDir: "docs/api",
             downloadUrl: "https://raw.githubusercontent.com/meta-llama/llama-stack/main/docs/static/llama-stack-spec.yaml",
@@ -179,6 +196,24 @@ const config: Config = {
               categoryLinkSource: "tag",
             },
           } satisfies OpenApiPlugin.Options,
+          experimental: {
+            specPath: "static/experimental-llama-stack-spec.yaml",
+            outputDir: "docs/api-experimental",
+            downloadUrl: "https://raw.githubusercontent.com/meta-llama/llama-stack/main/docs/static/experimental-llama-stack-spec.yaml",
+            sidebarOptions: {
+              groupPathsBy: "tag",
+              categoryLinkSource: "tag",
+            },
+          } satisfies OpenApiPlugin.Options,
+          deprecated: {
+            specPath: "static/deprecated-llama-stack-spec.yaml",
+            outputDir: "docs/api-deprecated",
+            downloadUrl: "https://raw.githubusercontent.com/meta-llama/llama-stack/main/docs/static/deprecated-llama-stack-spec.yaml",
+            sidebarOptions: {
+              groupPathsBy: "tag",
+              categoryLinkSource: "tag",
+            },
+          } satisfies OpenApiPlugin.Options,
         } satisfies Plugin.PluginOptions,
       },
     ],

docs/openapi_generator/generate.py

@@ -34,40 +34,52 @@ def str_presenter(dumper, data):
     return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style)
 
 
-def main(output_dir: str):
-    output_dir = Path(output_dir)
-    if not output_dir.exists():
-        raise ValueError(f"Directory {output_dir} does not exist")
+def generate_spec(output_dir: Path, stability_filter: str = None, main_spec: bool = False):
+    """Generate OpenAPI spec with optional stability filtering."""
 
-    # Validate API protocols before generating spec
-    return_type_errors = validate_api()
-    if return_type_errors:
-        print("\nAPI Method Return Type Validation Errors:\n")
-        for error in return_type_errors:
-            print(error, file=sys.stderr)
-        sys.exit(1)
-    now = str(datetime.now())
-    print(
-        "Converting the spec to YAML (openapi.yaml) and HTML (openapi.html) at " + now
-    )
-    print("")
+    if stability_filter:
+        title_suffix = {
+            "stable": " - Stable APIs" if not main_spec else "",
+            "experimental": " - Experimental APIs",
+            "deprecated": " - Deprecated APIs"
+        }.get(stability_filter, f" - {stability_filter.title()} APIs")
+
+        # Use main spec filename for stable when main_spec=True
+        if main_spec and stability_filter == "stable":
+            filename_prefix = ""
+        else:
+            filename_prefix = f"{stability_filter}-"
+
+        description_suffix = {
+            "stable": "\n\n**✅ STABLE**: Production-ready APIs with backward compatibility guarantees.",
+            "experimental": "\n\n**🧪 EXPERIMENTAL**: Pre-release APIs (v1alpha, v1beta) that may change before becoming stable.",
+            "deprecated": "\n\n**⚠️ DEPRECATED**: Legacy APIs that may be removed in future versions. Use for migration reference only."
+        }.get(stability_filter, "")
+    else:
+        title_suffix = ""
+        filename_prefix = ""
+        description_suffix = ""
 
     spec = Specification(
         LlamaStack,
         Options(
             server=Server(url="http://any-hosted-llama-stack.com"),
             info=Info(
-                title="Llama Stack Specification",
+                title=f"Llama Stack Specification{title_suffix}",
                 version=LLAMA_STACK_API_V1,
-                description="""This is the specification of the Llama Stack that provides
+                description=f"""This is the specification of the Llama Stack that provides
                 a set of endpoints and their corresponding interfaces that are tailored to
-                best leverage Llama Models.""",
+                best leverage Llama Models.{description_suffix}""",
             ),
             include_standard_error_responses=True,
+            stability_filter=stability_filter,  # Pass the filter to the generator
         ),
     )
 
-    with open(output_dir / "llama-stack-spec.yaml", "w", encoding="utf-8") as fp:
+    yaml_filename = f"{filename_prefix}llama-stack-spec.yaml"
+    html_filename = f"{filename_prefix}llama-stack-spec.html"
+
+    with open(output_dir / yaml_filename, "w", encoding="utf-8") as fp:
         y = yaml.YAML()
         y.default_flow_style = False
         y.block_seq_indent = 2
@@ -83,9 +95,36 @@ def main(output_dir: str):
             fp,
         )
 
-    with open(output_dir / "llama-stack-spec.html", "w") as fp:
+    with open(output_dir / html_filename, "w") as fp:
         spec.write_html(fp, pretty_print=True)
 
+    print(f"Generated {yaml_filename} and {html_filename}")
+
+def main(output_dir: str):
+    output_dir = Path(output_dir)
+    if not output_dir.exists():
+        raise ValueError(f"Directory {output_dir} does not exist")
+
+    # Validate API protocols before generating spec
+    return_type_errors = validate_api()
+    if return_type_errors:
+        print("\nAPI Method Return Type Validation Errors:\n")
+        for error in return_type_errors:
+            print(error, file=sys.stderr)
+        sys.exit(1)
+
+    now = str(datetime.now())
+    print(f"Converting the spec to YAML (openapi.yaml) and HTML (openapi.html) at {now}")
+    print("")
+
+    # Generate main spec as stable APIs (llama-stack-spec.yaml)
+    print("Generating main specification (stable APIs)...")
+    generate_spec(output_dir, "stable", main_spec=True)
+
+    print("Generating other stability-filtered specifications...")
+    generate_spec(output_dir, "experimental")
+    generate_spec(output_dir, "deprecated")
+
 
 if __name__ == "__main__":
     fire.Fire(main)