Add robots.txt to optimize AI crawler indexing for MLflow documentation (#386)

Co-authored-by: copilot-swe-agent[bot] <[email protected]>
Co-authored-by: kevin-lyn <[email protected]>

Author: Copilot

website/static/robots.txt

@@ -0,0 +1,59 @@
+# Robots.txt for MLflow Documentation
+# Optimized for AI crawlers to prioritize latest documentation
+
+# Default rules for all crawlers
+User-agent: *
+# Allow latest documentation
+Allow: /docs/latest/
+# Disallow all legacy documentation versions
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Specific rules for AI crawlers
+# OpenAI (ChatGPT)
+User-agent: ChatGPT-User
+User-agent: GPTBot
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Google Gemini
+User-agent: Google-Extended
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Anthropic Claude
+User-agent: ClaudeBot
+User-agent: Claude-Web
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Common Crawl (used by many AI systems)
+User-agent: CCBot
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Perplexity
+User-agent: PerplexityBot
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Cohere
+User-agent: cohere-ai
+Allow: /docs/latest/
+Disallow: /docs/1.*/
+Disallow: /docs/2.*/
+Disallow: /docs/0.*/
+
+# Sitemap location
+Sitemap: http://mlflow.org/sitemap.xml

website/tests/robots.spec.ts

@@ -0,0 +1,38 @@
+import { test, expect } from "@playwright/test";
+
+test.describe("Robots.txt", () => {
+  test("robots.txt is accessible", async ({ page }) => {
+    const response = await page.goto("/robots.txt");
+    expect(response?.status()).toBe(200);
+  });
+
+  test("robots.txt allows latest docs", async ({ page }) => {
+    const response = await page.goto("/robots.txt");
+    const content = await response?.text();
+    expect(content).toContain("Allow: /docs/latest/");
+  });
+
+  test("robots.txt disallows legacy versions", async ({ page }) => {
+    const response = await page.goto("/robots.txt");
+    const content = await response?.text();
+    expect(content).toContain("Disallow: /docs/1.*/");
+    expect(content).toContain("Disallow: /docs/2.*/");
+    expect(content).toContain("Disallow: /docs/0.*/");
+  });
+
+  test("robots.txt includes AI crawler configurations", async ({ page }) => {
+    const response = await page.goto("/robots.txt");
+    const content = await response?.text();
+    // Check for various AI crawlers
+    expect(content).toContain("GPTBot");
+    expect(content).toContain("ClaudeBot");
+    expect(content).toContain("Google-Extended");
+    expect(content).toContain("CCBot");
+  });
+
+  test("robots.txt includes sitemap", async ({ page }) => {
+    const response = await page.goto("/robots.txt");
+    const content = await response?.text();
+    expect(content).toContain("Sitemap:");
+  });
+});