From 97770070cb786a525815a577b8a7412716e87686 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Thu, 9 Oct 2025 09:59:48 -0700 Subject: [PATCH 1/7] WIP --- libs/arcade-cli/arcade_cli/deploy_server.py | 382 ++++++++++++++++++++ libs/arcade-cli/arcade_cli/main.py | 64 ++++ 2 files changed, 446 insertions(+) create mode 100644 libs/arcade-cli/arcade_cli/deploy_server.py diff --git a/libs/arcade-cli/arcade_cli/deploy_server.py b/libs/arcade-cli/arcade_cli/deploy_server.py new file mode 100644 index 000000000..a9b8098f0 --- /dev/null +++ b/libs/arcade-cli/arcade_cli/deploy_server.py @@ -0,0 +1,382 @@ +""" +Deploy MCP servers directly to Arcade Engine. + +This module handles the deployment of MCP servers to Arcade Engine via the /v1/deployments endpoint. +It is completely independent from the legacy arcade_cli.deployment module to allow for clean separation. +""" + +import base64 +import importlib.util +import io +import os +import sys +import tarfile +from pathlib import Path + +import httpx +from rich.console import Console + +from arcade_cli.utils import compute_base_url, validate_and_get_config + +console = Console() + + +def load_mcp_app_from_entrypoint(entrypoint: str) -> "MCPApp": # type: ignore + """ + Dynamically import the entrypoint file and extract the MCPApp instance. + + Args: + entrypoint: Relative path to the entrypoint file (e.g., "server.py") + + Returns: + The MCPApp instance found in the module + + Raises: + FileNotFoundError: If entrypoint doesn't exist + ValueError: If no MCPApp instance is found or multiple are found + """ + from arcade_mcp_server.mcp_app import MCPApp + + entrypoint_path = Path(entrypoint).resolve() + + if not entrypoint_path.exists(): + raise FileNotFoundError(f"Entrypoint file not found: {entrypoint}") + + if not entrypoint_path.is_file(): + raise ValueError(f"Entrypoint must be a file, not a directory: {entrypoint}") + + # Create a unique module name to avoid conflicts + module_name = f"_arcade_deploy_{entrypoint_path.stem}" + + # Load the module from the file path + spec = importlib.util.spec_from_file_location(module_name, entrypoint_path) + if spec is None or spec.loader is None: + raise ValueError(f"Could not load module spec from {entrypoint}") + + module = importlib.util.module_from_spec(spec) + + # Add to sys.modules temporarily so relative imports work + sys.modules[module_name] = module + + try: + # Execute the module + spec.loader.exec_module(module) + except Exception as e: + # Clean up sys.modules on error + sys.modules.pop(module_name, None) + raise ValueError(f"Failed to import entrypoint module: {e}") from e + + # Find all MCPApp instances in the module + mcp_apps = [] + for name, obj in vars(module).items(): + if isinstance(obj, MCPApp): + mcp_apps.append((name, obj)) + + # Clean up sys.modules + sys.modules.pop(module_name, None) + + if len(mcp_apps) == 0: + raise ValueError(f"No MCPApp instance found in {entrypoint}") + + if len(mcp_apps) > 1: + app_names = ", ".join(name for name, _ in mcp_apps) + raise ValueError( + f"Multiple MCPApp instances found in {entrypoint}: {app_names}. " + "Please ensure only one MCPApp instance is defined." + ) + + _, app = mcp_apps[0] + return app + + +def get_required_secrets(app: "MCPApp") -> set[str]: # type: ignore + """ + Extract all required secret keys from the MCPApp's catalog. + + Args: + app: The MCPApp instance + + Returns: + A set of secret key names required by all tools + """ + required_secrets = set() + + # Iterate through all tools in the catalog + for tool in app._catalog: + # Check if tool has secret requirements + if tool.definition.requirements and tool.definition.requirements.secrets: + for secret in tool.definition.requirements.secrets: + if secret.key: + required_secrets.add(secret.key) + + return required_secrets + + +def create_package_archive(package_dir: Path) -> str: + """ + Create a tar.gz archive of the package directory. + + Args: + package_dir: Path to the package directory to archive + + Returns: + Base64-encoded string of the tar.gz archive bytes + + Raises: + ValueError: If package_dir doesn't exist or is not a directory + """ + if not package_dir.exists(): + raise ValueError(f"Package directory not found: {package_dir}") + + if not package_dir.is_dir(): + raise ValueError(f"Package path must be a directory: {package_dir}") + + def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: + """Filter for files/directories to exclude from the archive.""" + name = tarinfo.name + + # Exclude hidden files and directories + parts = Path(name).parts + if any(part.startswith(".") for part in parts): + return None + + # Exclude __pycache__ directories + if "__pycache__" in parts: + return None + + # Exclude .egg-info directories + if any(part.endswith(".egg-info") for part in parts): + return None + + # Exclude dist and build directories + if "dist" in parts or "build" in parts: + return None + + # Exclude files ending with .lock + if name.endswith(".lock"): + return None + + return tarinfo + + # Create tar.gz archive in memory + byte_stream = io.BytesIO() + with tarfile.open(fileobj=byte_stream, mode="w:gz") as tar: + tar.add(package_dir, arcname=package_dir.name, filter=exclude_filter) + + # Get bytes and encode to base64 + byte_stream.seek(0) + package_bytes = byte_stream.read() + package_bytes_b64 = base64.b64encode(package_bytes).decode("utf-8") + + return package_bytes_b64 + + +def upsert_secrets_to_engine( + engine_url: str, api_key: str, secrets: set[str], debug: bool = False +) -> None: + """ + Upsert secrets to the Arcade Engine. + + Args: + engine_url: The base URL of the Arcade Engine + api_key: The API key for authentication + secrets: Set of secret keys to upsert + debug: Whether to show debug information + """ + if not secrets: + return + + console.print(f"\nRequired secrets: {', '.join(sorted(secrets))}", style="dim") + + client = httpx.Client(base_url=engine_url, headers={"Authorization": f"Bearer {api_key}"}) + + for secret_key in sorted(secrets): + secret_value = os.getenv(secret_key) + + if debug: + if secret_value: + console.print( + f" Found secret '{secret_key}' in environment (value ends with ...{secret_value[-4:]})", + style="dim", + ) + else: + console.print( + f" Secret '{secret_key}' not found in environment", style="dim yellow" + ) + + if not secret_value: + console.print( + f"⚠️ Secret '{secret_key}' not found in environment, skipping.", + style="yellow", + ) + continue + + try: + # Upsert secret to engine + response = client.put( + f"/v1/admin/secrets/{secret_key}", + json={"description": "Secret set via CLI", "value": secret_value}, + timeout=30, + ) + response.raise_for_status() + console.print(f"✓ Secret '{secret_key}' uploaded", style="dim green") + except httpx.HTTPStatusError as e: + error_msg = f"Failed to upload secret '{secret_key}': HTTP {e.response.status_code}" + if debug: + console.print(f"❌ {error_msg}: {e.response.text}", style="red") + else: + console.print(f"❌ {error_msg}", style="red") + except Exception as e: + error_msg = f"Failed to upload secret '{secret_key}': {e}" + console.print(f"❌ {error_msg}", style="red") + + client.close() + + +def deploy_server_to_engine( + engine_url: str, api_key: str, deployment_request: dict, debug: bool = False +) -> dict: + """ + Deploy the server to Arcade Engine. + + Args: + engine_url: The base URL of the Arcade Engine + api_key: The API key for authentication + deployment_request: The deployment request payload + debug: Whether to show debug information + + Returns: + The response JSON from the deployment API + + Raises: + httpx.HTTPStatusError: If the deployment request fails + httpx.ConnectError: If connection to the engine fails + """ + client = httpx.Client( + base_url=engine_url, + headers={"Authorization": f"Bearer {api_key}"}, + timeout=360.0, + ) + + try: + response = client.post("/v1/deployments", json=deployment_request) + response.raise_for_status() + return response.json() + except httpx.ConnectError as e: + raise ValueError(f"Failed to connect to Arcade Engine at {engine_url}: {e}") from e + except httpx.HTTPStatusError as e: + error_detail = "" + try: + error_json = e.response.json() + error_detail = f": {error_json}" + except Exception: + error_detail = f": {e.response.text}" + + raise ValueError( + f"Deployment failed with HTTP {e.response.status_code}{error_detail}" + ) from e + finally: + client.close() + + +def deploy_server_logic( + entrypoint: str, + host: str, + port: int | None, + force_tls: bool, + force_no_tls: bool, + debug: bool, +) -> None: + """ + Main logic for deploying an MCP server to Arcade Engine. + + Args: + entrypoint: Path to the entrypoint file containing MCPApp + host: Arcade Engine host + port: Arcade Engine port (optional) + force_tls: Force TLS connection + force_no_tls: Disable TLS connection + debug: Show debug information + """ + # Step 1: Validate user is logged in + config = validate_and_get_config() + engine_url = compute_base_url(force_tls, force_no_tls, host, port) + + # Step 2: Validate pyproject.toml exists in current directory + current_dir = Path.cwd() + pyproject_path = current_dir / "pyproject.toml" + + if not pyproject_path.exists(): + raise FileNotFoundError( + f"pyproject.toml not found in current directory: {current_dir}\n" + "Please run this command from the root of your MCP server package." + ) + + # Step 3: Load MCPApp from entrypoint + console.print(f"\nLoading MCP server from {entrypoint}...", style="dim") + try: + app = load_mcp_app_from_entrypoint(entrypoint) + except Exception as e: + raise ValueError(f"Failed to load MCPApp from {entrypoint}: {e}") from e + + # Step 4: Get server name and version from app + server_name = app.name + server_version = app.version + tool_count = len(list(app._catalog)) + + console.print(f"✓ Found server: {server_name} v{server_version}", style="green") + console.print(f" Discovered {tool_count} tool(s) in catalog", style="dim") + + # Step 5: Discover required secrets + required_secrets = get_required_secrets(app) + + # Step 6: Upsert secrets to engine + if required_secrets: + upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) + else: + console.print("\nNo secrets required", style="dim") + + # Step 7: Create tar.gz archive of current directory + console.print("\nCreating deployment package...", style="dim") + try: + archive_base64 = create_package_archive(current_dir) + # Calculate size in MB + archive_size_mb = len(archive_base64) * 3 / 4 / 1024 / 1024 # base64 is ~4/3 larger + console.print(f"✓ Package created ({archive_size_mb:.1f} MB)", style="green") + except Exception as e: + raise ValueError(f"Failed to create package archive: {e}") from e + + # Step 8: Build deployment request payload + deployment_request = { + "name": server_name, + "type": "mcp", + "entrypoint": entrypoint, + "description": "MCP Server deployed via CLI", + "toolkits": { + "bundles": [ + { + "name": server_name, + "version": server_version, + "bytes": archive_base64, + } + ] + }, + } + + # Step 9: Send deployment request to engine + console.print("\nDeploying to Arcade Engine...", style="dim") + try: + response = deploy_server_to_engine(engine_url, config.api.key, deployment_request, debug) + except Exception as e: + raise ValueError(f"Deployment failed: {e}") from e + + # Step 10: Display success message with deployment details + console.print( + f"✓ Server '{server_name}' v{server_version} deployed successfully", style="bold green" + ) + console.print(f"\nDeployment URL: {engine_url}/v1/deployments/{server_name}", style="dim") + console.print(f"Tools: {tool_count} tool(s) deployed", style="dim") + + if debug and response: + console.print("\nDeployment response:", style="dim") + console.print(response) diff --git a/libs/arcade-cli/arcade_cli/main.py b/libs/arcade-cli/arcade_cli/main.py index 5b536c9ff..e0b02716a 100644 --- a/libs/arcade-cli/arcade_cli/main.py +++ b/libs/arcade-cli/arcade_cli/main.py @@ -633,6 +633,70 @@ def deploy( handle_cli_error(f"Failed to deploy server '{worker.config.id}'", e, debug) +@cli.command( + name="deploy-server", help="Deploy MCP servers to Arcade Engine", rich_help_panel="Run" +) +def deploy_server( + entrypoint: str = typer.Option( + "./server.py", + "--entrypoint", + "-e", + help="Path to the Python file that contains the MCPApp instance (relative to project root)", + ), + host: str = typer.Option( + PROD_ENGINE_HOST, + "--host", + "-h", + help="The Arcade Engine host to deploy to", + hidden=True, + ), + port: Optional[int] = typer.Option( + None, + "--port", + "-p", + help="The port of the Arcade Engine", + hidden=True, + ), + force_tls: bool = typer.Option( + False, + "--tls", + help="Force TLS for the connection to the Arcade Engine", + hidden=True, + ), + force_no_tls: bool = typer.Option( + False, + "--no-tls", + help="Disable TLS for the connection to the Arcade Engine", + hidden=True, + ), + debug: bool = typer.Option(False, "--debug", "-d", help="Show debug information"), +) -> None: + """ + Deploy an MCP server directly to Arcade Engine. + + This command should be run from the root of your MCP server package + (the directory containing pyproject.toml). + + Examples: + cd my_mcp_server/ + arcade deploy-server + arcade deploy-server --entrypoint src/server.py + """ + from arcade_cli.deploy_server import deploy_server_logic + + try: + deploy_server_logic( + entrypoint=entrypoint, + host=host, + port=port, + force_tls=force_tls, + force_no_tls=force_no_tls, + debug=debug, + ) + except Exception as e: + handle_cli_error("Failed to deploy server", e, debug) + + @cli.command(help="Open the Arcade Dashboard in a web browser", rich_help_panel="User") def dashboard( host: str = typer.Option( From 4308f5110771d6b95fd10aed399970f07e5c2805 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Thu, 9 Oct 2025 15:12:50 -0700 Subject: [PATCH 2/7] Intermdiary commit --- libs/arcade-cli/arcade_cli/deploy_server.py | 266 +++++++++++++++++++- 1 file changed, 256 insertions(+), 10 deletions(-) diff --git a/libs/arcade-cli/arcade_cli/deploy_server.py b/libs/arcade-cli/arcade_cli/deploy_server.py index a9b8098f0..6937fde08 100644 --- a/libs/arcade-cli/arcade_cli/deploy_server.py +++ b/libs/arcade-cli/arcade_cli/deploy_server.py @@ -5,23 +5,106 @@ It is completely independent from the legacy arcade_cli.deployment module to allow for clean separation. """ +import ast import base64 import importlib.util import io import os +import subprocess import sys import tarfile +import time from pathlib import Path +from typing import TYPE_CHECKING import httpx from rich.console import Console from arcade_cli.utils import compute_base_url, validate_and_get_config +if TYPE_CHECKING: + from arcade_mcp_server.mcp_app import MCPApp + console = Console() -def load_mcp_app_from_entrypoint(entrypoint: str) -> "MCPApp": # type: ignore +def validate_entrypoint_structure(entrypoint: str) -> None: + """ + Validate that the entrypoint file has proper structure with if __name__ == "__main__". + + This ensures that app.run() or similar blocking calls are protected and won't + block the import when we load the MCPApp for inspection. + + Args: + entrypoint: Path to the entrypoint file + + Raises: + ValueError: If the file structure is invalid + """ + entrypoint_path = Path(entrypoint).resolve() + + try: + with open(entrypoint_path) as f: + source = f.read() + except Exception as e: + raise ValueError(f"Failed to read entrypoint file: {e}") from e + + try: + tree = ast.parse(source, filename=str(entrypoint_path)) + except SyntaxError as e: + raise ValueError(f"Syntax error in entrypoint file: {e}") from e + + # Check if there's an if __name__ == "__main__" block + has_main_block = False + + for node in ast.walk(tree): + # Check for if __name__ == "__main__" pattern + if isinstance(node, ast.If): + if _is_main_guard(node): + has_main_block = True + break + + if not has_main_block: + raise ValueError( + f"Entrypoint file '{entrypoint}' must have an 'if __name__ == \"__main__\"' block that runs the server.\n\n" + "The entrypoint should be structured like:\n\n" + " app = MCPApp(name='my_server', version='1.0.0')\n\n" + " @app.tool\n" + " def my_tool():\n" + " ...\n\n" + " if __name__ == '__main__':\n" + " app.run()\n\n" + ) + + +def _is_main_guard(node: ast.If) -> bool: + """Check if an If node is an if __name__ == '__main__' guard.""" + if not isinstance(node.test, ast.Compare): + return False + + compare = node.test + if not isinstance(compare.left, ast.Name): + return False + + if compare.left.id != "__name__": + return False + + if len(compare.ops) != 1 or not isinstance(compare.ops[0], ast.Eq): + return False + + if len(compare.comparators) != 1: + return False + + comparator = compare.comparators[0] + if isinstance(comparator, ast.Constant): + return comparator.value == "__main__" + elif isinstance(comparator, ast.Str): # Python 3.7 compatibility + return comparator.s == "__main__" + + return False + + +def load_mcp_app_from_entrypoint(entrypoint: str) -> "MCPApp": """ Dynamically import the entrypoint file and extract the MCPApp instance. @@ -171,6 +254,154 @@ def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: return package_bytes_b64 +def start_local_server(entrypoint: str, port: int = 8000) -> subprocess.Popen: + """ + Start the MCP server locally as a subprocess. + + Args: + entrypoint: Path to the entrypoint file + port: Port to run the server on + + Returns: + The subprocess handle + + Raises: + ValueError: If the server process fails to start + """ + # Use Python to run the entrypoint file + cmd = [sys.executable, entrypoint] + + # Start the process, capturing stdout and stderr + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={**os.environ, "PORT": str(port)}, + ) + + return process + + +def check_server_health(port: int = 8000, timeout: int = 30, check_interval: float = 0.5) -> bool: + """ + Poll the server's health endpoint until it responds or times out. + + Args: + port: Port the server is running on + timeout: Maximum time to wait in seconds + check_interval: Time between health checks in seconds + + Returns: + True if the server becomes healthy, False otherwise + """ + health_url = f"http://127.0.0.1:{port}/worker/health" + start_time = time.time() + + while time.time() - start_time < timeout: + try: + response = httpx.get(health_url, timeout=2.0) + if response.status_code == 200: + return True + except (httpx.ConnectError, httpx.TimeoutException): + # Server not ready yet, continue polling + pass + except Exception: + # Unexpected error, but continue polling + pass + + time.sleep(check_interval) + + return False + + +def stop_local_server(process: subprocess.Popen) -> None: + """ + Stop the local MCP server process. + + Args: + process: The subprocess handle + """ + try: + # Try graceful shutdown first + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + # Force kill if graceful shutdown fails + process.kill() + process.wait() + except Exception: + # If anything goes wrong, try to kill it + try: + process.kill() + except Exception: + pass + + +def verify_server_starts(entrypoint: str, port: int = 8000, debug: bool = False) -> None: + """ + Verify that the server starts and becomes healthy. + + Args: + entrypoint: Path to the entrypoint file + port: Port to run the server on + debug: Whether to show debug information + + Raises: + ValueError: If the server fails to start or become healthy + """ + console.print("\nVerifying server starts correctly...", style="dim") + + # Start the server + try: + process = start_local_server(entrypoint, port) + except Exception as e: + raise ValueError(f"Failed to start server process: {e}") from e + + # Check if process immediately failed + time.sleep(0.5) + if process.poll() is not None: + # Process already exited + _, stderr = process.communicate() + error_msg = stderr.strip() if stderr else "Unknown error" + raise ValueError(f"Server process exited immediately: {error_msg}") + + # Poll health endpoint + try: + is_healthy = check_server_health(port=port, timeout=30) + + if not is_healthy: + # Server didn't become healthy in time + stop_local_server(process) + + # Try to get error output + try: + _, stderr = process.communicate(timeout=2) + error_msg = stderr.strip() if stderr else "Server failed to become healthy" + except subprocess.TimeoutExpired: + error_msg = "Server failed to become healthy within 30 seconds" + + raise ValueError(error_msg) + + # Server is healthy! + console.print("✓ Server started successfully and is healthy", style="green") + + # Stop the server + stop_local_server(process) + + if debug: + console.print(" Server verification complete, process stopped", style="dim") + + except ValueError: + # Re-raise verification errors + raise + except Exception as e: + # Cleanup and raise + stop_local_server(process) + raise ValueError(f"Error during server verification: {e}") from e + + def upsert_secrets_to_engine( engine_url: str, api_key: str, secrets: set[str], debug: bool = False ) -> None: @@ -255,7 +486,7 @@ def deploy_server_to_engine( client = httpx.Client( base_url=engine_url, headers={"Authorization": f"Bearer {api_key}"}, - timeout=360.0, + timeout=360, ) try: @@ -312,14 +543,29 @@ def deploy_server_logic( "Please run this command from the root of your MCP server package." ) - # Step 3: Load MCPApp from entrypoint + # Step 3: Validate entrypoint structure + try: + validate_entrypoint_structure(entrypoint) + except Exception as e: + raise ValueError(f"Invalid entrypoint structure: {e}") from e + + # Step 4: Verify server starts and is healthy + try: + verify_server_starts(entrypoint, port=8000, debug=debug) + except Exception as e: + raise ValueError( + f"Server verification failed: {e}\n" + "Please ensure your server starts correctly before deploying." + ) from e + + # Step 5: Load MCPApp from entrypoint console.print(f"\nLoading MCP server from {entrypoint}...", style="dim") try: app = load_mcp_app_from_entrypoint(entrypoint) except Exception as e: raise ValueError(f"Failed to load MCPApp from {entrypoint}: {e}") from e - # Step 4: Get server name and version from app + # Step 6: Get server name and version from app server_name = app.name server_version = app.version tool_count = len(list(app._catalog)) @@ -327,16 +573,16 @@ def deploy_server_logic( console.print(f"✓ Found server: {server_name} v{server_version}", style="green") console.print(f" Discovered {tool_count} tool(s) in catalog", style="dim") - # Step 5: Discover required secrets + # Step 7: Discover required secrets required_secrets = get_required_secrets(app) - # Step 6: Upsert secrets to engine + # Step 8: Upsert secrets to engine if required_secrets: upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) else: console.print("\nNo secrets required", style="dim") - # Step 7: Create tar.gz archive of current directory + # Step 9: Create tar.gz archive of current directory console.print("\nCreating deployment package...", style="dim") try: archive_base64 = create_package_archive(current_dir) @@ -346,7 +592,7 @@ def deploy_server_logic( except Exception as e: raise ValueError(f"Failed to create package archive: {e}") from e - # Step 8: Build deployment request payload + # Step 10: Build deployment request payload deployment_request = { "name": server_name, "type": "mcp", @@ -363,14 +609,14 @@ def deploy_server_logic( }, } - # Step 9: Send deployment request to engine + # Step 11: Send deployment request to engine console.print("\nDeploying to Arcade Engine...", style="dim") try: response = deploy_server_to_engine(engine_url, config.api.key, deployment_request, debug) except Exception as e: raise ValueError(f"Deployment failed: {e}") from e - # Step 10: Display success message with deployment details + # Step 12: Display success message with deployment details console.print( f"✓ Server '{server_name}' v{server_version} deployed successfully", style="bold green" ) From 92e92fda55f1933065a2ba1147bfae4ad8fee1e1 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Thu, 9 Oct 2025 16:23:46 -0700 Subject: [PATCH 3/7] Dont import module --- libs/arcade-cli/arcade_cli/deploy_server.py | 450 +++++++------------- 1 file changed, 145 insertions(+), 305 deletions(-) diff --git a/libs/arcade-cli/arcade_cli/deploy_server.py b/libs/arcade-cli/arcade_cli/deploy_server.py index 6937fde08..20e1fe6c1 100644 --- a/libs/arcade-cli/arcade_cli/deploy_server.py +++ b/libs/arcade-cli/arcade_cli/deploy_server.py @@ -5,196 +5,26 @@ It is completely independent from the legacy arcade_cli.deployment module to allow for clean separation. """ -import ast import base64 -import importlib.util import io import os +import random import subprocess import sys import tarfile import time from pathlib import Path -from typing import TYPE_CHECKING import httpx +from arcade_core.config_model import Config +from dotenv import load_dotenv from rich.console import Console from arcade_cli.utils import compute_base_url, validate_and_get_config -if TYPE_CHECKING: - from arcade_mcp_server.mcp_app import MCPApp - console = Console() -def validate_entrypoint_structure(entrypoint: str) -> None: - """ - Validate that the entrypoint file has proper structure with if __name__ == "__main__". - - This ensures that app.run() or similar blocking calls are protected and won't - block the import when we load the MCPApp for inspection. - - Args: - entrypoint: Path to the entrypoint file - - Raises: - ValueError: If the file structure is invalid - """ - entrypoint_path = Path(entrypoint).resolve() - - try: - with open(entrypoint_path) as f: - source = f.read() - except Exception as e: - raise ValueError(f"Failed to read entrypoint file: {e}") from e - - try: - tree = ast.parse(source, filename=str(entrypoint_path)) - except SyntaxError as e: - raise ValueError(f"Syntax error in entrypoint file: {e}") from e - - # Check if there's an if __name__ == "__main__" block - has_main_block = False - - for node in ast.walk(tree): - # Check for if __name__ == "__main__" pattern - if isinstance(node, ast.If): - if _is_main_guard(node): - has_main_block = True - break - - if not has_main_block: - raise ValueError( - f"Entrypoint file '{entrypoint}' must have an 'if __name__ == \"__main__\"' block that runs the server.\n\n" - "The entrypoint should be structured like:\n\n" - " app = MCPApp(name='my_server', version='1.0.0')\n\n" - " @app.tool\n" - " def my_tool():\n" - " ...\n\n" - " if __name__ == '__main__':\n" - " app.run()\n\n" - ) - - -def _is_main_guard(node: ast.If) -> bool: - """Check if an If node is an if __name__ == '__main__' guard.""" - if not isinstance(node.test, ast.Compare): - return False - - compare = node.test - if not isinstance(compare.left, ast.Name): - return False - - if compare.left.id != "__name__": - return False - - if len(compare.ops) != 1 or not isinstance(compare.ops[0], ast.Eq): - return False - - if len(compare.comparators) != 1: - return False - - comparator = compare.comparators[0] - if isinstance(comparator, ast.Constant): - return comparator.value == "__main__" - elif isinstance(comparator, ast.Str): # Python 3.7 compatibility - return comparator.s == "__main__" - - return False - - -def load_mcp_app_from_entrypoint(entrypoint: str) -> "MCPApp": - """ - Dynamically import the entrypoint file and extract the MCPApp instance. - - Args: - entrypoint: Relative path to the entrypoint file (e.g., "server.py") - - Returns: - The MCPApp instance found in the module - - Raises: - FileNotFoundError: If entrypoint doesn't exist - ValueError: If no MCPApp instance is found or multiple are found - """ - from arcade_mcp_server.mcp_app import MCPApp - - entrypoint_path = Path(entrypoint).resolve() - - if not entrypoint_path.exists(): - raise FileNotFoundError(f"Entrypoint file not found: {entrypoint}") - - if not entrypoint_path.is_file(): - raise ValueError(f"Entrypoint must be a file, not a directory: {entrypoint}") - - # Create a unique module name to avoid conflicts - module_name = f"_arcade_deploy_{entrypoint_path.stem}" - - # Load the module from the file path - spec = importlib.util.spec_from_file_location(module_name, entrypoint_path) - if spec is None or spec.loader is None: - raise ValueError(f"Could not load module spec from {entrypoint}") - - module = importlib.util.module_from_spec(spec) - - # Add to sys.modules temporarily so relative imports work - sys.modules[module_name] = module - - try: - # Execute the module - spec.loader.exec_module(module) - except Exception as e: - # Clean up sys.modules on error - sys.modules.pop(module_name, None) - raise ValueError(f"Failed to import entrypoint module: {e}") from e - - # Find all MCPApp instances in the module - mcp_apps = [] - for name, obj in vars(module).items(): - if isinstance(obj, MCPApp): - mcp_apps.append((name, obj)) - - # Clean up sys.modules - sys.modules.pop(module_name, None) - - if len(mcp_apps) == 0: - raise ValueError(f"No MCPApp instance found in {entrypoint}") - - if len(mcp_apps) > 1: - app_names = ", ".join(name for name, _ in mcp_apps) - raise ValueError( - f"Multiple MCPApp instances found in {entrypoint}: {app_names}. " - "Please ensure only one MCPApp instance is defined." - ) - - _, app = mcp_apps[0] - return app - - -def get_required_secrets(app: "MCPApp") -> set[str]: # type: ignore - """ - Extract all required secret keys from the MCPApp's catalog. - - Args: - app: The MCPApp instance - - Returns: - A set of secret key names required by all tools - """ - required_secrets = set() - - # Iterate through all tools in the catalog - for tool in app._catalog: - # Check if tool has secret requirements - if tool.definition.requirements and tool.definition.requirements.secrets: - for secret in tool.definition.requirements.secrets: - if secret.key: - required_secrets.add(secret.key) - - return required_secrets - - def create_package_archive(package_dir: Path) -> str: """ Create a tar.gz archive of the package directory. @@ -254,152 +84,176 @@ def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: return package_bytes_b64 -def start_local_server(entrypoint: str, port: int = 8000) -> subprocess.Popen: +def verify_server_and_get_metadata( + entrypoint: str, debug: bool = False +) -> tuple[str, str, set[str]]: """ - Start the MCP server locally as a subprocess. + Start the server, verify it's healthy, and extract metadata. + + This function: + 1. Picks a random port + 2. Starts the server with environment variables set + 3. Waits for the server to become healthy + 4. Extracts server name and version via POST /mcp + 5. Extracts required secrets via GET /worker/tools + 6. Stops the server + 7. Returns the metadata Args: entrypoint: Path to the entrypoint file - port: Port to run the server on + debug: Whether to show debug information Returns: - The subprocess handle + Tuple of (server_name, server_version, required_secrets_set) Raises: - ValueError: If the server process fails to start + ValueError: If the server fails to start or metadata extraction fails """ - # Use Python to run the entrypoint file - cmd = [sys.executable, entrypoint] + console.print("\nVerifying server and extracting metadata...", style="dim") - # Start the process, capturing stdout and stderr + port = random.randint(8000, 9000) # noqa: S311 + + # Set environment variables to override app.run() settings + env = { + **os.environ, + "ARCADE_SERVER_HOST": "localhost", + "ARCADE_SERVER_PORT": str(port), + "ARCADE_SERVER_TRANSPORT": "http", + "ARCADE_AUTH_DISABLED": "true", + } + + # Start the server + cmd = [sys.executable, entrypoint] process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, - env={**os.environ, "PORT": str(port)}, + env=env, ) - return process - + if debug: + console.print(f" Started server on port {port}", style="dim") -def check_server_health(port: int = 8000, timeout: int = 30, check_interval: float = 0.5) -> bool: - """ - Poll the server's health endpoint until it responds or times out. - - Args: - port: Port the server is running on - timeout: Maximum time to wait in seconds - check_interval: Time between health checks in seconds + # Check if process immediately failed + time.sleep(0.5) + if process.poll() is not None: + _, stderr = process.communicate() + error_msg = stderr.strip() if stderr else "Unknown error" + raise ValueError(f"Server process exited immediately: {error_msg}") - Returns: - True if the server becomes healthy, False otherwise - """ - health_url = f"http://127.0.0.1:{port}/worker/health" + # Poll health endpoint + base_url = f"http://127.0.0.1:{port}" + health_url = f"{base_url}/worker/health" start_time = time.time() + is_healthy = False - while time.time() - start_time < timeout: + while time.time() - start_time < 30: try: response = httpx.get(health_url, timeout=2.0) if response.status_code == 200: - return True + is_healthy = True + break except (httpx.ConnectError, httpx.TimeoutException): - # Server not ready yet, continue polling pass except Exception: - # Unexpected error, but continue polling pass + time.sleep(0.5) - time.sleep(check_interval) - - return False - - -def stop_local_server(process: subprocess.Popen) -> None: - """ - Stop the local MCP server process. - - Args: - process: The subprocess handle - """ - try: - # Try graceful shutdown first + if not is_healthy: + # Server didn't become healthy process.terminate() try: - process.wait(timeout=5) + _, stderr = process.communicate(timeout=2) + error_msg = stderr.strip() if stderr else "Server failed to become healthy" except subprocess.TimeoutExpired: - # Force kill if graceful shutdown fails process.kill() - process.wait() - except Exception: - # If anything goes wrong, try to kill it - try: - process.kill() - except Exception: - pass - + error_msg = "Server failed to become healthy within 30 seconds" + raise ValueError(error_msg) -def verify_server_starts(entrypoint: str, port: int = 8000, debug: bool = False) -> None: - """ - Verify that the server starts and becomes healthy. - - Args: - entrypoint: Path to the entrypoint file - port: Port to run the server on - debug: Whether to show debug information + console.print("✓ Server is healthy", style="green") - Raises: - ValueError: If the server fails to start or become healthy - """ - console.print("\nVerifying server starts correctly...", style="dim") - - # Start the server try: - process = start_local_server(entrypoint, port) - except Exception as e: - raise ValueError(f"Failed to start server process: {e}") from e + # Extract server name and version via POST /mcp + mcp_url = f"{base_url}/mcp" + initialize_request = { + "id": 1, + "jsonrpc": "2.0", + "method": "initialize", + "params": { + "capabilities": {}, + "clientInfo": {"name": "arcade-deploy-client", "version": "1.0.0"}, + "protocolVersion": "2025-06-18", + }, + } + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } - # Check if process immediately failed - time.sleep(0.5) - if process.poll() is not None: - # Process already exited - _, stderr = process.communicate() - error_msg = stderr.strip() if stderr else "Unknown error" - raise ValueError(f"Server process exited immediately: {error_msg}") + try: + mcp_response = httpx.post( + mcp_url, json=initialize_request, headers=headers, timeout=10.0 + ) + mcp_response.raise_for_status() + mcp_data = mcp_response.json() - # Poll health endpoint - try: - is_healthy = check_server_health(port=port, timeout=30) + server_name = mcp_data["result"]["serverInfo"]["name"] + server_version = mcp_data["result"]["serverInfo"]["version"] - if not is_healthy: - # Server didn't become healthy in time - stop_local_server(process) + if debug: + console.print( + f" Extracted name: {server_name}, version: {server_version}", style="dim" + ) + except Exception as e: + raise ValueError(f"Failed to extract server info from /mcp endpoint: {e}") from e + + # Extract required secrets via GET /worker/tools + tools_url = f"{base_url}/worker/tools" - # Try to get error output - try: - _, stderr = process.communicate(timeout=2) - error_msg = stderr.strip() if stderr else "Server failed to become healthy" - except subprocess.TimeoutExpired: - error_msg = "Server failed to become healthy within 30 seconds" + try: + tools_response = httpx.get(tools_url, timeout=10.0) + tools_response.raise_for_status() + tools_data = tools_response.json() + + required_secrets = set() + for tool in tools_data: + if ( + "requirements" in tool + and tool["requirements"] + and "secrets" in tool["requirements"] + and tool["requirements"]["secrets"] + ): + for secret in tool["requirements"]["secrets"]: + if "key" in secret and secret["key"]: + required_secrets.add(secret["key"]) - raise ValueError(error_msg) + if debug: + if required_secrets: + console.print( + f" Found {len(required_secrets)} required secret(s)", style="dim" + ) + else: + console.print(" No secrets required", style="dim") - # Server is healthy! - console.print("✓ Server started successfully and is healthy", style="green") + except Exception as e: + raise ValueError( + f"Failed to extract tool secrets from /worker/tools endpoint: {e}" + ) from e - # Stop the server - stop_local_server(process) + finally: + # Always stop the server + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() if debug: - console.print(" Server verification complete, process stopped", style="dim") + console.print(" Server stopped", style="dim") - except ValueError: - # Re-raise verification errors - raise - except Exception as e: - # Cleanup and raise - stop_local_server(process) - raise ValueError(f"Error during server verification: {e}") from e + return server_name, server_version, required_secrets def upsert_secrets_to_engine( @@ -543,56 +397,43 @@ def deploy_server_logic( "Please run this command from the root of your MCP server package." ) - # Step 3: Validate entrypoint structure - try: - validate_entrypoint_structure(entrypoint) - except Exception as e: - raise ValueError(f"Invalid entrypoint structure: {e}") from e + # Step 2.5: Load .env file from current directory if it exists + env_path = current_dir / ".env" + if env_path.exists(): + load_dotenv(env_path, override=False) + if debug: + console.print(f" Loaded environment from {env_path}", style="dim") - # Step 4: Verify server starts and is healthy + # Step 3: Verify server and extract metadata try: - verify_server_starts(entrypoint, port=8000, debug=debug) + server_name, server_version, required_secrets = verify_server_and_get_metadata( + entrypoint, debug=debug + ) except Exception as e: raise ValueError( f"Server verification failed: {e}\n" "Please ensure your server starts correctly before deploying." ) from e - # Step 5: Load MCPApp from entrypoint - console.print(f"\nLoading MCP server from {entrypoint}...", style="dim") - try: - app = load_mcp_app_from_entrypoint(entrypoint) - except Exception as e: - raise ValueError(f"Failed to load MCPApp from {entrypoint}: {e}") from e - - # Step 6: Get server name and version from app - server_name = app.name - server_version = app.version - tool_count = len(list(app._catalog)) - console.print(f"✓ Found server: {server_name} v{server_version}", style="green") - console.print(f" Discovered {tool_count} tool(s) in catalog", style="dim") - - # Step 7: Discover required secrets - required_secrets = get_required_secrets(app) - # Step 8: Upsert secrets to engine + # Step 4: Upsert secrets to engine if required_secrets: + console.print(f"\nDiscovered {len(required_secrets)} required secret(s)", style="dim") upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) else: console.print("\nNo secrets required", style="dim") - # Step 9: Create tar.gz archive of current directory + # Step 5: Create tar.gz archive of current directory console.print("\nCreating deployment package...", style="dim") try: archive_base64 = create_package_archive(current_dir) - # Calculate size in MB - archive_size_mb = len(archive_base64) * 3 / 4 / 1024 / 1024 # base64 is ~4/3 larger - console.print(f"✓ Package created ({archive_size_mb:.1f} MB)", style="green") + archive_size_kb = len(archive_base64) * 3 / 4 / 1024 # base64 is ~4/3 larger + console.print(f"✓ Package created ({archive_size_kb:.1f} KB)", style="green") except Exception as e: raise ValueError(f"Failed to create package archive: {e}") from e - # Step 10: Build deployment request payload + # Step 6: Build deployment request payload deployment_request = { "name": server_name, "type": "mcp", @@ -609,19 +450,18 @@ def deploy_server_logic( }, } - # Step 11: Send deployment request to engine + # Step 7: Send deployment request to engine console.print("\nDeploying to Arcade Engine...", style="dim") try: response = deploy_server_to_engine(engine_url, config.api.key, deployment_request, debug) except Exception as e: raise ValueError(f"Deployment failed: {e}") from e - # Step 12: Display success message with deployment details + # Step 8: Display success message with deployment details console.print( f"✓ Server '{server_name}' v{server_version} deployed successfully", style="bold green" ) console.print(f"\nDeployment URL: {engine_url}/v1/deployments/{server_name}", style="dim") - console.print(f"Tools: {tool_count} tool(s) deployed", style="dim") if debug and response: console.print("\nDeployment response:", style="dim") From 1fa5b040f08e7e9e6ac220af6d18415a45522108 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Sun, 12 Oct 2025 20:26:30 -0700 Subject: [PATCH 4/7] Working E2E --- .../{deploy_server.py => deploy.py} | 372 ++++++++----- libs/arcade-cli/arcade_cli/deployment.py | 500 ------------------ libs/arcade-cli/arcade_cli/main.py | 118 +---- libs/tests/cli/deploy/test_deploy.py | 362 +++++++++++++ .../invalid_server/pyproject.toml | 37 ++ .../test_servers/invalid_server/server.py | 73 +++ .../test_servers/valid_server/pyproject.toml | 37 ++ .../test_servers/valid_server/server.py | 73 +++ libs/tests/deployment/test_config.py | 231 -------- .../test_files/env.secret.worker.toml | 5 - .../deployment/test_files/full.worker.toml | 26 - .../test_files/invalid.fields.worker.toml | 3 - .../test_files/invalid.localfile.worker.toml | 42 -- .../test_files/invalid.secret.worker.toml | 7 - .../invalid_toolkit/invalid_main.py | 0 .../test_files/mock_toolkit/mock_main.py | 1 - .../test_files/mock_toolkit/pyproject.toml | 5 - 17 files changed, 831 insertions(+), 1061 deletions(-) rename libs/arcade-cli/arcade_cli/{deploy_server.py => deploy.py} (59%) delete mode 100644 libs/arcade-cli/arcade_cli/deployment.py create mode 100644 libs/tests/cli/deploy/test_deploy.py create mode 100644 libs/tests/cli/deploy/test_servers/invalid_server/pyproject.toml create mode 100644 libs/tests/cli/deploy/test_servers/invalid_server/server.py create mode 100644 libs/tests/cli/deploy/test_servers/valid_server/pyproject.toml create mode 100644 libs/tests/cli/deploy/test_servers/valid_server/server.py delete mode 100644 libs/tests/deployment/test_config.py delete mode 100644 libs/tests/deployment/test_files/env.secret.worker.toml delete mode 100644 libs/tests/deployment/test_files/full.worker.toml delete mode 100644 libs/tests/deployment/test_files/invalid.fields.worker.toml delete mode 100644 libs/tests/deployment/test_files/invalid.localfile.worker.toml delete mode 100644 libs/tests/deployment/test_files/invalid.secret.worker.toml delete mode 100644 libs/tests/deployment/test_files/invalid_toolkit/invalid_main.py delete mode 100644 libs/tests/deployment/test_files/mock_toolkit/mock_main.py delete mode 100644 libs/tests/deployment/test_files/mock_toolkit/pyproject.toml diff --git a/libs/arcade-cli/arcade_cli/deploy_server.py b/libs/arcade-cli/arcade_cli/deploy.py similarity index 59% rename from libs/arcade-cli/arcade_cli/deploy_server.py rename to libs/arcade-cli/arcade_cli/deploy.py index 20e1fe6c1..c60731fc6 100644 --- a/libs/arcade-cli/arcade_cli/deploy_server.py +++ b/libs/arcade-cli/arcade_cli/deploy.py @@ -1,10 +1,3 @@ -""" -Deploy MCP servers directly to Arcade Engine. - -This module handles the deployment of MCP servers to Arcade Engine via the /v1/deployments endpoint. -It is completely independent from the legacy arcade_cli.deployment module to allow for clean separation. -""" - import base64 import io import os @@ -14,16 +7,71 @@ import tarfile import time from pathlib import Path +from typing import cast import httpx -from arcade_core.config_model import Config from dotenv import load_dotenv +from pydantic import BaseModel, Field from rich.console import Console from arcade_cli.utils import compute_base_url, validate_and_get_config console = Console() +# Models + + +class MCPClientInfo(BaseModel): + """MCP client information for initialize request.""" + + name: str + version: str + + +class MCPInitializeParams(BaseModel): + """Parameters for MCP initialize request.""" + + capabilities: dict = Field(default_factory=dict) + clientInfo: MCPClientInfo + protocolVersion: str + + +class MCPInitializeRequest(BaseModel): + """MCP initialize request payload.""" + + id: int + jsonrpc: str = "2.0" + method: str = "initialize" + params: MCPInitializeParams + + +class ToolkitBundle(BaseModel): + """A toolkit bundle for deployment.""" + + name: str + version: str + bytes: str + type: str = "mcp" + entrypoint: str + + +class DeploymentToolkits(BaseModel): + """Toolkits section of deployment request.""" + + bundles: list[ToolkitBundle] + packages: list[str] = Field(default_factory=list) + + +class DeploymentRequest(BaseModel): + """Deployment request payload for /v1/deployments endpoint.""" + + name: str + description: str + toolkits: DeploymentToolkits + + +# Functions + def create_package_archive(package_dir: Path) -> str: """ @@ -45,27 +93,30 @@ def create_package_archive(package_dir: Path) -> str: raise ValueError(f"Package path must be a directory: {package_dir}") def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: - """Filter for files/directories to exclude from the archive.""" + """Filter for files/directories to exclude from the archive. + + Filters out: + - Hidden files and directories + - __pycache__ directories + - .egg-info directories + - dist and build directories + - files ending with .lock + """ name = tarinfo.name - # Exclude hidden files and directories parts = Path(name).parts if any(part.startswith(".") for part in parts): return None - # Exclude __pycache__ directories if "__pycache__" in parts: return None - # Exclude .egg-info directories if any(part.endswith(".egg-info") for part in parts): return None - # Exclude dist and build directories if "dist" in parts or "build" in parts: return None - # Exclude files ending with .lock if name.endswith(".lock"): return None @@ -84,36 +135,23 @@ def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: return package_bytes_b64 -def verify_server_and_get_metadata( - entrypoint: str, debug: bool = False -) -> tuple[str, str, set[str]]: +def start_server_process(entrypoint: str, debug: bool = False) -> tuple[subprocess.Popen, int]: """ - Start the server, verify it's healthy, and extract metadata. - - This function: - 1. Picks a random port - 2. Starts the server with environment variables set - 3. Waits for the server to become healthy - 4. Extracts server name and version via POST /mcp - 5. Extracts required secrets via GET /worker/tools - 6. Stops the server - 7. Returns the metadata + Start the MCP server process on a random port. Args: - entrypoint: Path to the entrypoint file + entrypoint: Path to the entrypoint file that runs the MCPApp instance debug: Whether to show debug information Returns: - Tuple of (server_name, server_version, required_secrets_set) + Tuple of (process, port) Raises: - ValueError: If the server fails to start or metadata extraction fails + ValueError: If the server process exits immediately """ - console.print("\nVerifying server and extracting metadata...", style="dim") - port = random.randint(8000, 9000) # noqa: S311 - # Set environment variables to override app.run() settings + # override app.run() settings env = { **os.environ, "ARCADE_SERVER_HOST": "localhost", @@ -122,7 +160,6 @@ def verify_server_and_get_metadata( "ARCADE_AUTH_DISABLED": "true", } - # Start the server cmd = [sys.executable, entrypoint] process = subprocess.Popen( cmd, @@ -135,20 +172,33 @@ def verify_server_and_get_metadata( if debug: console.print(f" Started server on port {port}", style="dim") - # Check if process immediately failed + # Check for immediate failure on start up time.sleep(0.5) if process.poll() is not None: _, stderr = process.communicate() error_msg = stderr.strip() if stderr else "Unknown error" raise ValueError(f"Server process exited immediately: {error_msg}") - # Poll health endpoint - base_url = f"http://127.0.0.1:{port}" + return process, port + + +def wait_for_health(base_url: str, process: subprocess.Popen, timeout: int = 30) -> None: + """ + Wait for the server to become healthy. + + Args: + base_url: Base URL of the server + process: The server process + timeout: Maximum time to wait in seconds + + Raises: + ValueError: If the server doesn't become healthy within timeout + """ health_url = f"{base_url}/worker/health" start_time = time.time() is_healthy = False - while time.time() - start_time < 30: + while time.time() - start_time < timeout: try: response = httpx.get(health_url, timeout=2.0) if response.status_code == 200: @@ -157,89 +207,153 @@ def verify_server_and_get_metadata( except (httpx.ConnectError, httpx.TimeoutException): pass except Exception: - pass + console.print(" Health check failed. Trying again...", style="dim") time.sleep(0.5) if not is_healthy: - # Server didn't become healthy process.terminate() try: _, stderr = process.communicate(timeout=2) error_msg = stderr.strip() if stderr else "Server failed to become healthy" except subprocess.TimeoutExpired: process.kill() - error_msg = "Server failed to become healthy within 30 seconds" + error_msg = f"Server failed to become healthy within {timeout} seconds" raise ValueError(error_msg) console.print("✓ Server is healthy", style="green") + +def get_server_info(base_url: str) -> tuple[str, str]: + """ + Extract server name and version via the MCP initialize endpoint. + + Args: + base_url: Base URL of the server + + Returns: + Tuple of (server_name, server_version) + + Raises: + ValueError: If server info extraction fails + """ + mcp_url = f"{base_url}/mcp" + + initialize_request = MCPInitializeRequest( + id=1, + params=MCPInitializeParams( + clientInfo=MCPClientInfo(name="arcade-deploy-client", version="1.0.0"), + protocolVersion="2025-06-18", + ), + ) + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + } + try: - # Extract server name and version via POST /mcp - mcp_url = f"{base_url}/mcp" - initialize_request = { - "id": 1, - "jsonrpc": "2.0", - "method": "initialize", - "params": { - "capabilities": {}, - "clientInfo": {"name": "arcade-deploy-client", "version": "1.0.0"}, - "protocolVersion": "2025-06-18", - }, - } - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - } + mcp_response = httpx.post( + mcp_url, json=initialize_request.model_dump(), headers=headers, timeout=10.0 + ) + mcp_response.raise_for_status() + mcp_data = mcp_response.json() - try: - mcp_response = httpx.post( - mcp_url, json=initialize_request, headers=headers, timeout=10.0 - ) - mcp_response.raise_for_status() - mcp_data = mcp_response.json() + server_name = mcp_data["result"]["serverInfo"]["name"] + server_version = mcp_data["result"]["serverInfo"]["version"] - server_name = mcp_data["result"]["serverInfo"]["name"] - server_version = mcp_data["result"]["serverInfo"]["version"] + console.print(f"✓ Found server: {server_name} v{server_version}", style="green") - if debug: - console.print( - f" Extracted name: {server_name}, version: {server_version}", style="dim" - ) - except Exception as e: - raise ValueError(f"Failed to extract server info from /mcp endpoint: {e}") from e + except Exception as e: + raise ValueError(f"Failed to extract server info from /mcp endpoint: {e}") from e + else: + return server_name, server_version - # Extract required secrets via GET /worker/tools - tools_url = f"{base_url}/worker/tools" - try: - tools_response = httpx.get(tools_url, timeout=10.0) - tools_response.raise_for_status() - tools_data = tools_response.json() - - required_secrets = set() - for tool in tools_data: - if ( - "requirements" in tool - and tool["requirements"] - and "secrets" in tool["requirements"] - and tool["requirements"]["secrets"] - ): - for secret in tool["requirements"]["secrets"]: - if "key" in secret and secret["key"]: - required_secrets.add(secret["key"]) +def get_required_secrets( + base_url: str, server_name: str, server_version: str, debug: bool = False +) -> set[str]: + """ + Extract required secrets from the /worker/tools endpoint. - if debug: - if required_secrets: - console.print( - f" Found {len(required_secrets)} required secret(s)", style="dim" - ) - else: - console.print(" No secrets required", style="dim") + Args: + base_url: Base URL of the server + server_name: Name of the server (for display purposes) + server_version: Version of the server (for display purposes) + debug: Whether to show debug information - except Exception as e: - raise ValueError( - f"Failed to extract tool secrets from /worker/tools endpoint: {e}" - ) from e + Returns: + Set of required secret keys + + Raises: + ValueError: If secrets extraction fails + """ + tools_url = f"{base_url}/worker/tools" + + try: + tools_response = httpx.get(tools_url, timeout=10.0) + tools_response.raise_for_status() + tools_data = tools_response.json() + + required_secrets = set() + for tool in tools_data: + if ( + "requirements" in tool + and tool["requirements"] + and "secrets" in tool["requirements"] + and tool["requirements"]["secrets"] + ): + for secret in tool["requirements"]["secrets"]: + if secret.get("key"): + required_secrets.add(secret["key"]) + + console.print(f"{server_name} v{server_version} has {len(tools_data)} tools", style="dim") + if debug: + if required_secrets: + console.print(f" Found {len(required_secrets)} required secret(s)", style="dim") + else: + console.print(" No secrets required", style="dim") + + except Exception as e: + raise ValueError(f"Failed to extract tool secrets from /worker/tools endpoint: {e}") from e + else: + return required_secrets + + +def verify_server_and_get_metadata( + entrypoint: str, debug: bool = False +) -> tuple[str, str, set[str]]: + """ + Start the server, verify it's healthy, and extract metadata. + + This function orchestrates: + 1. Starting the server on a random port + 2. Waiting for the server to become healthy + 3. Extracting server name and version via POST /mcp (initialize method) + 4. Extracting required secrets via GET /worker/tools + 5. Stopping the server + 6. Returning the metadata + + Args: + entrypoint: Path to the entrypoint file that runs the MCPApp instance + debug: Whether to show debug information + + Returns: + Tuple of (server_name, server_version, required_secrets_set) + + Raises: + ValueError: If the server fails to start or metadata extraction fails + """ + console.print("\nVerifying server and extracting metadata...", style="dim") + + process, port = start_server_process(entrypoint, debug) + base_url = f"http://127.0.0.1:{port}" + + try: + wait_for_health(base_url, process) + server_name, server_version = get_server_info(base_url) + required_secrets = get_required_secrets(base_url, server_name, server_version, debug) + + return server_name, server_version, required_secrets finally: # Always stop the server @@ -253,8 +367,6 @@ def verify_server_and_get_metadata( if debug: console.print(" Server stopped", style="dim") - return server_name, server_version, required_secrets - def upsert_secrets_to_engine( engine_url: str, api_key: str, secrets: set[str], debug: bool = False @@ -346,7 +458,7 @@ def deploy_server_to_engine( try: response = client.post("/v1/deployments", json=deployment_request) response.raise_for_status() - return response.json() + return cast(dict, response.json()) except httpx.ConnectError as e: raise ValueError(f"Failed to connect to Arcade Engine at {engine_url}: {e}") from e except httpx.HTTPStatusError as e: @@ -397,14 +509,14 @@ def deploy_server_logic( "Please run this command from the root of your MCP server package." ) - # Step 2.5: Load .env file from current directory if it exists + # Step 3: Load .env file from current directory if it exists env_path = current_dir / ".env" if env_path.exists(): load_dotenv(env_path, override=False) if debug: console.print(f" Loaded environment from {env_path}", style="dim") - # Step 3: Verify server and extract metadata + # Step 4: Verify server and extract metadata try: server_name, server_version, required_secrets = verify_server_and_get_metadata( entrypoint, debug=debug @@ -415,16 +527,14 @@ def deploy_server_logic( "Please ensure your server starts correctly before deploying." ) from e - console.print(f"✓ Found server: {server_name} v{server_version}", style="green") - - # Step 4: Upsert secrets to engine + # Step 5: Upsert secrets to engine if required_secrets: console.print(f"\nDiscovered {len(required_secrets)} required secret(s)", style="dim") upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) else: console.print("\nNo secrets required", style="dim") - # Step 5: Create tar.gz archive of current directory + # Step 6: Create tar.gz archive of current directory console.print("\nCreating deployment package...", style="dim") try: archive_base64 = create_package_archive(current_dir) @@ -433,35 +543,39 @@ def deploy_server_logic( except Exception as e: raise ValueError(f"Failed to create package archive: {e}") from e - # Step 6: Build deployment request payload - deployment_request = { - "name": server_name, - "type": "mcp", - "entrypoint": entrypoint, - "description": "MCP Server deployed via CLI", - "toolkits": { - "bundles": [ - { - "name": server_name, - "version": server_version, - "bytes": archive_base64, - } - ] - }, - } + # Step 7: Build deployment request payload + deployment_request = DeploymentRequest( + name=server_name, + description="MCP Server deployed via CLI", + toolkits=DeploymentToolkits( + bundles=[ + ToolkitBundle( + name=server_name, + version=server_version, + bytes=archive_base64, + type="mcp", + entrypoint=entrypoint, + ) + ], + ), + ) - # Step 7: Send deployment request to engine + # Step 8: Send deployment request to engine console.print("\nDeploying to Arcade Engine...", style="dim") try: - response = deploy_server_to_engine(engine_url, config.api.key, deployment_request, debug) + response = deploy_server_to_engine( + engine_url, config.api.key, deployment_request.model_dump(), debug + ) except Exception as e: raise ValueError(f"Deployment failed: {e}") from e - # Step 8: Display success message with deployment details console.print( f"✓ Server '{server_name}' v{server_version} deployed successfully", style="bold green" ) - console.print(f"\nDeployment URL: {engine_url}/v1/deployments/{server_name}", style="dim") + console.print( + "\nView your deployed servers in the Arcade Dashboard: https://api.arcade.dev/dashboard/", + style="dim", + ) if debug and response: console.print("\nDeployment response:", style="dim") diff --git a/libs/arcade-cli/arcade_cli/deployment.py b/libs/arcade-cli/arcade_cli/deployment.py deleted file mode 100644 index 2787f2428..000000000 --- a/libs/arcade-cli/arcade_cli/deployment.py +++ /dev/null @@ -1,500 +0,0 @@ -import base64 -import io -import os -import re -import secrets -import tarfile -import time -from pathlib import Path -from typing import Any - -import toml -from arcade_core import Toolkit -from arcade_core.catalog import ToolCatalog -from arcade_core.toolkit import Validate -from arcadepy import Arcade, NotFoundError -from httpx import Client, ConnectError, HTTPStatusError, TimeoutException -from packaging.requirements import Requirement -from pydantic import BaseModel, field_serializer, field_validator, model_validator -from rich.console import Console -from rich.table import Table - -console = Console() - - -# Base class for versioned packages -class Package(BaseModel): - name: str - specifier: str | None = None - - @classmethod - def from_requirement(cls, requirement_str: str) -> "Package": - req = Requirement(requirement_str) - return cls(name=req.name, specifier=str(req.specifier) if req.specifier else None) - - -# Base class for a list of packages -class Packages(BaseModel): - packages: list[Package] - - # Convert string package i.e. "arcade>1.0.0" to a name and specifier - # Specifiers are currently unused - @field_validator("packages", mode="before") - @classmethod - def parse_package_requirements(cls, packages: list[str]) -> list[Package]: - """Convert package requirement strings to Package objects.""" - return [Package.from_requirement(pkg) for pkg in packages] - - -# Base class for a local package -class LocalPackage(BaseModel): - name: str - content: str - - -# Base class for a list of local packages -class LocalPackages(BaseModel): - packages: list[str] - - -# Custom repository configurations -class PackageRepository(Packages): - index: str - index_url: str - trusted_host: str - - -# Pypi is a special case of a package repository -class Pypi(PackageRepository): - index: str = "pypi" - index_url: str = "https://pypi.org/simple" - trusted_host: str = "pypi.org" - - -class Secret(BaseModel): - value: str - pattern: str | None = None - - -class AuthProvider(BaseModel): - """Configuration for a local auth provider.""" - - provider_id: str - """The provider ID (e.g., 'google', 'github', 'custom-oauth')""" - - provider_type: str = "oauth2" - """The type of provider, usually 'oauth2'""" - - client_id: str - """OAuth client ID for this provider""" - - client_secret: str - """OAuth client secret for this provider""" - - # Mock tokens for local development - mock_tokens: dict[str, str] | None = None - """ - Mock access tokens by user ID for local development. - Example: {"user-123": "mock-google-token-abc", "user-456": "mock-google-token-def"} - """ - - scopes: list[str] | None = None - """Default scopes for this provider""" - - -class Config(BaseModel): - """The configuration for an Arcade worker deployment.""" - - id: str - """The unique id for the worker deployment.""" - - enabled: bool = True - """Whether the worker is enabled. Defaults to True.""" - - secret: Secret | None = None - """The shared secret between the worker and Arcade Engine server.""" - - timeout: int = 120 - """The maximum execution time in seconds for a tool in this worker.""" - - retries: int = 1 - """The number of times to retry a failed tool invocation. Defaults to 1.""" - - # Local development context - only used when running locally - local_context: dict[str, Any] | None = None - """ - Local context configuration for development. This section is only used when running - 'arcade serve' locally and is ignored during deployment. It can include: - - user_id: Default user ID for local testing - - user_info: Dictionary of user metadata - - metadata: Additional metadata fields - Example: - [worker.config.local_context] - user_id = "test-user-123" - user_info = { email = "test@example.com", name = "Test User" } - """ - - # Local auth providers - only used when running locally - local_auth_providers: list[AuthProvider] | None = None - """ - Local auth provider configurations for development. These are only used when running - 'arcade serve' locally and are ignored during deployment. They define mock OAuth - providers and tokens for testing tools that require authentication. - Example: - [[worker.config.local_auth_providers]] - provider_id = "google" - client_id = "mock-google-client" - client_secret = "mock-google-secret" - [worker.config.local_auth_providers.mock_tokens] - "test-user-123" = "mock-google-access-token" - """ - - # Validate and parse the secret if required - @field_validator("secret", mode="before") - @classmethod - def valid_secret(cls, v: str | Secret | None) -> Secret: - # If the secret is a string, attempt to parse it as an environment variable or return the secret - if isinstance(v, str): - secret = get_env_secret(v) - elif isinstance(v, Secret): - secret = v - else: - raise TypeError("Secret must be a string or a Secret object") - if secret.value.strip() == "": - raise ValueError("Secret must be a non-empty string") - return secret - - @field_serializer("secret") - def serialize_secret(self, secret: Secret) -> str: - if secret.pattern: - return f"$env:{secret.pattern}" - return secret.value - - -# Cloud request for deploying a worker -class Request(BaseModel): - name: str - secret: Secret - enabled: bool - timeout: int - retries: int - pypi: Pypi | None = None - custom_repositories: list[PackageRepository] | None = None - local_packages: list[LocalPackage] | None = None - wait: bool = False - - @field_serializer("secret") - def serialize_secret(self, secret: Secret) -> str: - return secret.value - - def poll_worker_status(self, cloud_client: Client, worker_name: str) -> Any: - while True: - try: - worker_resp = cloud_client.get( - f"{cloud_client.base_url}/api/v1/workers/{worker_name}?wait_for_completion=true", - timeout=10, - ) - worker_resp.raise_for_status() - except TimeoutException: - time.sleep(1) - continue - except ConnectError as e: - raise ValueError(f"Failed to connect to cloud: {e}") - except HTTPStatusError as e: - raise ValueError(f"Failed to start worker: {e.response.json()}") - except Exception as e: - raise ValueError(f"Failed to start worker: {e}") - status = worker_resp.json()["data"]["status"] - if status == "Running": - return worker_resp.json()["data"] - if status == "Failed": - raise ValueError(f"Worker failed to start: {worker_resp.json()['data']['error']}") - - def execute(self, cloud_client: Client, engine_client: Arcade) -> Any: - # Attempt to deploy worker to the cloud - try: - cloud_response = cloud_client.put( - str(cloud_client.base_url) + "/api/v1/workers", - json=self.model_dump(mode="json"), - timeout=360, - ) - cloud_response.raise_for_status() - except ConnectError as e: - raise ValueError(f"Failed to connect to cloud: {e}") - except HTTPStatusError as e: - raise ValueError(f"Failed to start worker: {e.response.json()}") - except Exception as e: - # change this so it handles errors that aren't just from cloud - raise ValueError(f"Failed to start worker: {e}") - - parse_deployment_response(cloud_response.json()) - worker_data = self.poll_worker_status(cloud_client, self.name) - - try: - # Check if worker already exists - engine_client.workers.get(self.name) - engine_client.workers.update( - id=self.name, - enabled=self.enabled, - http={ - "uri": worker_data["endpoint"], - "secret": self.secret.value, - "timeout": self.timeout, - "retry": self.retries, - }, - ) - # If worker does not exist, create it - except NotFoundError: - engine_client.workers.create( - id=self.name, - enabled=self.enabled, - http={ - "uri": worker_data["endpoint"], - "secret": self.secret.value, - "timeout": self.timeout, - "retry": self.retries, - }, - ) - - except Exception as e: - raise ValueError(f"Failed to add worker to engine: {e}") - - return cloud_response.json() - - -class Worker(BaseModel): - toml_path: Path - config: Config - pypi_source: Pypi | None = None - custom_source: list[PackageRepository] | None = None - local_source: LocalPackages | None = None - - def request(self) -> Request: - """Convert Deployment to a Request object.""" - self.validate_packages() - self.compress_local_packages() - if self.config.secret is None: - raise ValueError("Secret is required") - return Request( - name=self.config.id, - secret=self.config.secret, - enabled=self.config.enabled, - timeout=self.config.timeout, - retries=self.config.retries, - pypi=self.pypi_source, - custom_repositories=self.custom_source, - local_packages=self.compress_local_packages(), - ) - - # Search for local packages and compress the source code to send - def compress_local_packages(self) -> list[LocalPackage] | None: - """Compress local packages into a list of LocalPackage objects.""" - if self.local_source is None: - return None - - def exclude_filter(tarinfo: tarfile.TarInfo) -> tarfile.TarInfo | None: - """Filter for files/directories to exclude from the compressed package""" - if not Validate.path(tarinfo.name): - return None - - return tarinfo - - # Compress local packages into a list of LocalPackage objects - def process_package(package_path_str: str) -> LocalPackage: - package_path = self.toml_path.parent / package_path_str - - if not package_path.exists(): - raise FileNotFoundError(f"Local package not found: {package_path}") - if not package_path.is_dir(): - raise FileNotFoundError(f"Local package is not a directory: {package_path}") - - # Check that the package is a valid python package - if ( - not (package_path / "pyproject.toml").is_file() - and not (package_path / "setup.py").is_file() - ): - raise ValueError( - f"package '{package_path}' must contain a pyproject.toml or setup.py file" - ) - - # Validate that we are able to load the package - # Use from_directory to properly resolve src/ layouts and avoid double prefixes - Toolkit.from_directory(package_path) - - # Compress the package into a byte stream and tar - byte_stream = io.BytesIO() - with tarfile.open(fileobj=byte_stream, mode="w:gz") as tar: - tar.add(package_path, arcname=package_path.name, filter=exclude_filter) - - byte_stream.seek(0) - package_bytes = byte_stream.read() - package_bytes_b64 = base64.b64encode(package_bytes).decode("utf-8") - - return LocalPackage(name=package_path.name, content=package_bytes_b64) - - return list(map(process_package, self.local_source.packages)) - - # Validate that there are no duplicate packages for each worker - def validate_packages(self) -> None: - """Validate packages.""" - packages: list[str] = [] - if self.pypi_source: - for pypi_package in self.pypi_source.packages: - packages.append(pypi_package.name) - if self.custom_source: - for repository in self.custom_source: - for package in repository.packages: - packages.append(package.name) - if self.local_source: - for local_package in self.local_source.packages: - packages.append(os.path.normpath(Path(local_package))) - dupes = [x for n, x in enumerate(packages) if x in packages[:n]] - if dupes: - raise ValueError(f"Duplicate packages: {dupes}") - - def get_required_secrets(self) -> set[str]: - """Inspect local toolkits and return a set of required secret keys.""" - all_secrets = set() - if self.local_source: - catalog = ToolCatalog() - for package_path_str in self.local_source.packages: - package_path = self.toml_path.parent / package_path_str - toolkit = Toolkit.from_directory(package_path) - catalog.add_toolkit(toolkit) - - for tool in catalog: - if tool.definition.requirements and tool.definition.requirements.secrets: - for secret in tool.definition.requirements.secrets: - all_secrets.add(secret.key) - return all_secrets - - -class Deployment(BaseModel): - toml_path: Path - worker: list[Worker] - - # Validate that there are no duplicate worker names - @model_validator(mode="after") - def validate_workers(self) -> "Deployment": - for worker in self.worker: - if sum(worker.config.id == w.config.id for w in self.worker) > 1: - raise ValueError(f"Duplicate worker name: {worker.config.id}") - return self - - # Load a deployment from a toml file - @classmethod - def from_toml(cls, toml_path: Path) -> "Deployment": - try: - with open(toml_path) as f: - toml_data = toml.load(f) - - if not toml_data: - raise ValueError(f"Empty TOML file: {toml_path}") - - # Add the toml path to each worker so relative packages can be found - if "worker" in toml_data: - for worker in toml_data["worker"]: - worker["toml_path"] = toml_path - - return cls(**toml_data, toml_path=toml_path) - - except toml.TomlDecodeError as e: - raise ValueError(f"Invalid TOML format in {toml_path}: {e!s}") - except FileNotFoundError: - raise FileNotFoundError(f"Config file not found: {toml_path}") - - # Save the deployment to a toml file - def save(self) -> None: - print("writing deployment file", self.toml_path) - with open(self.toml_path, "w") as f: - data = self.model_dump() - # Remove the toml_path from the deployment file - del data["toml_path"] - for worker in data["worker"]: - del worker["toml_path"] - toml.dump(data, f) - - -# Create a demo deployment file with one worker -def create_demo_deployment(toml_path: Path, toolkit_name: str) -> None: - """Create a deployment from a toml file.""" - deployment = Deployment( - toml_path=toml_path, - worker=[ - Worker( - toml_path=toml_path, - config=Config( - id="demo-worker", - enabled=True, - timeout=30, - retries=3, - secret=Secret(value=secrets.token_hex(16), pattern=None), - ), - local_source=LocalPackages(packages=[f"./{toolkit_name}"]), - ) - ], - ) - deployment.save() - - -# Get a currently existing deployment and add an additional local package -def update_deployment_with_local_packages(toml_path: Path, toolkit_name: str) -> None: - """Update a deployment from a toml file.""" - deployment = Deployment.from_toml(toml_path) - if deployment.worker[0].local_source is None: - deployment.worker[0].local_source = LocalPackages(packages=[f"./{toolkit_name}"]) - else: - deployment.worker[0].local_source.packages.append(f"./{toolkit_name}") - deployment.save() - - -def get_env_secret(secret: str) -> Secret: - """Parse a secret from an environment variable.""" - # Check if the secret contains the "${env:}" syntax - pattern = r"\${env:([^}]+)}" - matches = re.findall(pattern, secret) - - # Only allow a single match - if matches and len(matches) == 1: - match = matches[0].strip() - # Attempt to lookup and create the secret - print(f"Looking up secret: {match}") - value = os.getenv(match) - if value: - return Secret(value=value, pattern=match) - else: - raise ValueError(f"Environment variable not found: {match}") - elif matches and len(matches) > 1: - raise ValueError(f"Multiple environment variables found in secret: {secret}") - # If no matches are found, return the secret as is - return Secret(value=secret, pattern=None) - - -def parse_deployment_response(response: dict) -> None: - # Check what changes were made to the worker and display - changes = response["data"]["changes"] - additions = changes.get("additions", []) - removals = changes.get("removals", []) - updates = changes.get("updates", []) - no_changes = changes.get("no_changes", []) - print_deployment_table(additions, removals, updates, no_changes) - - -def print_deployment_table( - additions: list, removals: list, updates: list, no_changes: list -) -> None: - table = Table(title="Changed Packages") - table.add_column("Adding", justify="right", style="green") - table.add_column("Removing", justify="right", style="red") - table.add_column("Updating", justify="right", style="yellow") - table.add_column("No Changes", justify="right", style="dim") - max_rows = max(len(additions), len(removals), len(updates), len(no_changes)) - - # Add each row of worker package changes to the table - for i in range(max_rows): - addition = additions[i] if i < len(additions) else "" - removal = removals[i] if i < len(removals) else "" - update = updates[i] if i < len(updates) else "" - no_change = no_changes[i] if i < len(no_changes) else "" - table.add_row(addition, removal, update, no_change) - console.print(table) diff --git a/libs/arcade-cli/arcade_cli/main.py b/libs/arcade-cli/arcade_cli/main.py index e0b02716a..f9618dd55 100644 --- a/libs/arcade-cli/arcade_cli/main.py +++ b/libs/arcade-cli/arcade_cli/main.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Optional -import httpx import typer from arcadepy import Arcade from rich.console import Console @@ -23,7 +22,6 @@ PROD_CLOUD_HOST, PROD_ENGINE_HOST, ) -from arcade_cli.deployment import Deployment from arcade_cli.display import ( display_eval_results, ) @@ -531,117 +529,13 @@ def configure( handle_cli_error(f"Failed to configure {client}", e, debug) -@cli.command(help="Deploy servers to Arcade Cloud", rich_help_panel="Run", hidden=True) +@cli.command(name="deploy", help="Deploy MCP servers to Arcade", rich_help_panel="Run") def deploy( - deployment_file: str = typer.Option( - "worker.toml", - "--deployment-file", - "-d", - help="The deployment file to deploy.", - ), - cloud_host: str = typer.Option( - PROD_CLOUD_HOST, - "--cloud-host", - "-c", - help="The Arcade Cloud host to deploy to.", - hidden=True, - ), - cloud_port: Optional[int] = typer.Option( - None, - "--cloud-port", - "-cp", - help="The port of the Arcade Cloud host.", - hidden=True, - ), - host: str = typer.Option( - PROD_ENGINE_HOST, - "--host", - "-h", - help="The Arcade Engine host to register the server to.", - ), - port: Optional[int] = typer.Option( - None, - "--port", - "-p", - help="The port of the Arcade Engine host.", - ), - force_tls: bool = typer.Option( - False, - "--tls", - help="Whether to force TLS for the connection to the Arcade Engine. If not specified, the connection will use TLS if the engine URL uses a 'https' scheme.", - ), - force_no_tls: bool = typer.Option( - False, - "--no-tls", - help="Whether to disable TLS for the connection to the Arcade Engine.", - ), - debug: bool = typer.Option(False, "--debug", help="Show debug information"), -) -> None: - """ - Deploy a server to Arcade Cloud. - """ - - config = validate_and_get_config() - engine_url = compute_base_url(force_tls, force_no_tls, host, port) - engine_client = Arcade(api_key=config.api.key, base_url=engine_url) - cloud_url = compute_base_url(force_tls, force_no_tls, cloud_host, cloud_port) - cloud_client = httpx.Client( - base_url=cloud_url, headers={"Authorization": f"Bearer {config.api.key}"} - ) - - # Fetch deployment configuration - try: - deployment = Deployment.from_toml(Path(deployment_file)) - except Exception as e: - handle_cli_error("Failed to parse deployment file", e, debug) - - with console.status(f"Deploying {len(deployment.worker)} servers"): - for worker in deployment.worker: - console.log(f"Deploying '{worker.config.id}...'", style="dim") - try: - # Discover and upload secrets - required_secret_keys = worker.get_required_secrets() - for secret_key in required_secret_keys: - secret_value = os.getenv(secret_key) - if not secret_value: - console.log( - f"⚠️ Secret '{secret_key}' not found in environment, skipping.", - style="yellow", - ) - continue - try: - secret._upsert_secret_to_engine( - engine_url, config.api.key, secret_key, secret_value - ) - except Exception as e: - handle_cli_error( - f"Failed to upload secret '{secret_key}'", e, debug, should_exit=False - ) - else: - console.log( - f"✅ Secret '{secret_key}' uploaded successfully", - style="dim green", - ) - - # Attempt to deploy worker - worker.request().execute(cloud_client, engine_client) - console.log( - f"✅ Server '{worker.config.id}' deployed successfully.", - style="dim", - ) - except Exception as e: - handle_cli_error(f"Failed to deploy server '{worker.config.id}'", e, debug) - - -@cli.command( - name="deploy-server", help="Deploy MCP servers to Arcade Engine", rich_help_panel="Run" -) -def deploy_server( entrypoint: str = typer.Option( - "./server.py", + "server.py", "--entrypoint", "-e", - help="Path to the Python file that contains the MCPApp instance (relative to project root)", + help="Relative path to the Python file that runs the MCPApp instance (relative to project root)", ), host: str = typer.Option( PROD_ENGINE_HOST, @@ -679,10 +573,10 @@ def deploy_server( Examples: cd my_mcp_server/ - arcade deploy-server - arcade deploy-server --entrypoint src/server.py + arcade deploy + arcade deploy --entrypoint src/server.py """ - from arcade_cli.deploy_server import deploy_server_logic + from arcade_cli.deploy import deploy_server_logic try: deploy_server_logic( diff --git a/libs/tests/cli/deploy/test_deploy.py b/libs/tests/cli/deploy/test_deploy.py new file mode 100644 index 000000000..5f8619b56 --- /dev/null +++ b/libs/tests/cli/deploy/test_deploy.py @@ -0,0 +1,362 @@ +import base64 +import io +import subprocess +import tarfile +import time +from pathlib import Path + +import pytest +from arcade_cli.deploy import ( + create_package_archive, + get_required_secrets, + get_server_info, + start_server_process, + verify_server_and_get_metadata, + wait_for_health, +) + +# Fixtures + + +@pytest.fixture +def test_dir() -> Path: + """Return the path to the test directory.""" + return Path(__file__).parent + + +@pytest.fixture +def valid_server_dir(test_dir: Path) -> Path: + """Return the path to the valid server directory.""" + return test_dir / "test_servers" / "valid_server" + + +@pytest.fixture +def valid_server_path(valid_server_dir: Path) -> str: + """Return the path to the valid server entrypoint.""" + return str(valid_server_dir / "server.py") + + +@pytest.fixture +def invalid_server_path(test_dir: Path) -> str: + """Return the path to the invalid server entrypoint.""" + return str(test_dir / "test_servers" / "invalid_server" / "server.py") + + +@pytest.fixture +def tmp_project_dir(tmp_path: Path) -> Path: + """Create a temporary project directory with pyproject.toml.""" + project_dir = tmp_path / "test_project" + project_dir.mkdir() + + # Create a basic pyproject.toml + pyproject_content = """[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "test_project" +version = "0.1.0" +description = "Test project" +requires-python = ">=3.10" +""" + (project_dir / "pyproject.toml").write_text(pyproject_content) + return project_dir + + +# Tests for create_package_archive + + +def test_create_package_archive_success(valid_server_dir: Path) -> None: + """Test creating an archive from a valid directory.""" + archive_base64 = create_package_archive(valid_server_dir) + + # Verify it returns a base64-encoded string + assert isinstance(archive_base64, str) + assert len(archive_base64) > 0 + + # Decode and verify the archive can be extracted + archive_bytes = base64.b64decode(archive_base64) + byte_stream = io.BytesIO(archive_bytes) + + with tarfile.open(fileobj=byte_stream, mode="r:gz") as tar: + members = tar.getmembers() + filenames = [m.name for m in members] + + # Verify expected files are present + assert any("server.py" in name for name in filenames) + assert any("pyproject.toml" in name for name in filenames) + + +def test_create_package_archive_nonexistent_dir(tmp_path: Path) -> None: + """Test that archiving a non-existent directory raises ValueError.""" + nonexistent_dir = tmp_path / "does_not_exist" + + with pytest.raises(ValueError, match="Package directory not found"): + create_package_archive(nonexistent_dir) + + +def test_create_package_archive_file_not_dir(tmp_path: Path) -> None: + """Test that archiving a file instead of directory raises ValueError.""" + test_file = tmp_path / "test_file.txt" + test_file.write_text("test content") + + with pytest.raises(ValueError, match="Package path must be a directory"): + create_package_archive(test_file) + + +def test_create_package_archive_excludes_files(tmp_path: Path) -> None: + """Test that certain files are excluded from the archive.""" + test_dir = tmp_path / "test_project" + test_dir.mkdir() + + # Create files that should be excluded + (test_dir / ".hidden").write_text("hidden") + (test_dir / "__pycache__").mkdir() + (test_dir / "__pycache__" / "cache.pyc").write_text("cache") + (test_dir / "requirements.lock").write_text("lock") + (test_dir / "dist").mkdir() + (test_dir / "dist" / "package.tar.gz").write_text("dist") + (test_dir / "build").mkdir() + (test_dir / "build" / "lib").write_text("build") + + # Create files that should be included + (test_dir / "main.py").write_text("main") + (test_dir / "pyproject.toml").write_text("project") + + archive_base64 = create_package_archive(test_dir) + archive_bytes = base64.b64decode(archive_base64) + byte_stream = io.BytesIO(archive_bytes) + + with tarfile.open(fileobj=byte_stream, mode="r:gz") as tar: + members = tar.getmembers() + filenames = [m.name for m in members] + + # Verify excluded files are not present + assert not any(".hidden" in name for name in filenames) + assert not any("__pycache__" in name for name in filenames) + assert not any(".lock" in name for name in filenames) + assert not any("dist" in name for name in filenames) + assert not any("build" in name for name in filenames) + + # Verify included files are present + assert any("main.py" in name for name in filenames) + assert any("pyproject.toml" in name for name in filenames) + + +# Tests for start_server_process + + +def test_start_server_process_success(valid_server_path: str) -> None: + """Test starting a valid server process.""" + process, port = start_server_process(valid_server_path, debug=False) + + try: + # Verify process is running + assert process.poll() is None, "Process should be running" + + # Verify port is in expected range + assert 8000 <= port <= 9000 + + # Give the process a moment to start + time.sleep(1) + + # Verify process is still running + assert process.poll() is None, "Process should still be running" + finally: + # Clean up + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +# Tests for wait_for_health + + +def test_wait_for_health_success(valid_server_path: str, capsys) -> None: + """Test waiting for a healthy server.""" + process, port = start_server_process(valid_server_path, debug=False) + base_url = f"http://127.0.0.1:{port}" + + try: + # This should succeed without raising + wait_for_health(base_url, process, timeout=10) + + # Verify success message was printed + captured = capsys.readouterr() + assert "Server is healthy" in captured.out + finally: + # Clean up + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def test_wait_for_health_process_dies(valid_server_path: str) -> None: + """Test handling when process dies during health check.""" + process, port = start_server_process(valid_server_path, debug=False) + base_url = f"http://127.0.0.1:{port}" + + # Kill the process immediately + process.kill() + process.wait() + + # Mock process object to pass to wait_for_health + with pytest.raises(ValueError, match="Server failed to become healthy"): + wait_for_health(base_url, process, timeout=2) + + +# Tests for get_server_info + + +def test_get_server_info_success(valid_server_path: str, capsys) -> None: + """Test extracting server info from a running server.""" + process, port = start_server_process(valid_server_path, debug=False) + base_url = f"http://127.0.0.1:{port}" + + try: + # Wait for server to be healthy first + wait_for_health(base_url, process, timeout=10) + + # Get server info + server_name, server_version = get_server_info(base_url) + + # Verify expected values + assert server_name == "simpleserver" + assert server_version == "1.0.0" + + # Verify success message was printed + captured = capsys.readouterr() + assert "Found server: simpleserver v1.0.0" in captured.out + finally: + # Clean up + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def test_get_server_info_invalid_url() -> None: + """Test that invalid URL raises ValueError.""" + invalid_url = "http://127.0.0.1:9999" + + with pytest.raises(ValueError, match="Failed to extract server info from /mcp endpoint"): + get_server_info(invalid_url) + + +# Tests for get_required_secrets + + +def test_get_required_secrets_with_secrets(valid_server_path: str, capsys) -> None: + """Test extracting required secrets from server tools.""" + process, port = start_server_process(valid_server_path, debug=False) + base_url = f"http://127.0.0.1:{port}" + + try: + # Wait for server to be healthy first + wait_for_health(base_url, process, timeout=10) + + # Get required secrets + secrets = get_required_secrets(base_url, "simpleserver", "1.0.0", debug=True) + + # Verify expected secrets + assert "MY_SECRET_KEY" in secrets + + # Verify console output + captured = capsys.readouterr() + assert "simpleserver v1.0.0 has 3 tools" in captured.out + assert "Found 1 required secret(s)" in captured.out + finally: + # Clean up + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def test_get_required_secrets_no_secrets(valid_server_path: str) -> None: + """Test getting secrets returns set even when checking actual tools.""" + process, port = start_server_process(valid_server_path, debug=False) + base_url = f"http://127.0.0.1:{port}" + + try: + # Wait for server to be healthy first + wait_for_health(base_url, process, timeout=10) + + # Get required secrets (even though server has secrets, function should work) + secrets = get_required_secrets(base_url, "simpleserver", "1.0.0", debug=False) + + # Verify it returns a set + assert isinstance(secrets, set) + finally: + # Clean up + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def test_get_required_secrets_invalid_url() -> None: + """Test that invalid URL raises ValueError.""" + invalid_url = "http://127.0.0.1:9999" + + with pytest.raises( + ValueError, match="Failed to extract tool secrets from /worker/tools endpoint" + ): + get_required_secrets(invalid_url, "test", "1.0.0") + + +# Tests for verify_server_and_get_metadata (integration tests) + + +def test_verify_server_and_get_metadata_success(valid_server_path: str, capsys) -> None: + """Test full server verification flow.""" + server_name, server_version, required_secrets = verify_server_and_get_metadata( + valid_server_path, debug=False + ) + + # Verify returned values + assert server_name == "simpleserver" + assert server_version == "1.0.0" + assert "MY_SECRET_KEY" in required_secrets + + # Verify console output + captured = capsys.readouterr() + assert "Verifying server and extracting metadata" in captured.out + assert "Server is healthy" in captured.out + assert "Found server: simpleserver v1.0.0" in captured.out + + +def test_verify_server_and_get_metadata_invalid_server(invalid_server_path: str) -> None: + """Test that invalid server raises ValueError with cleanup.""" + with pytest.raises(ValueError, match="Server process exited immediately"): + verify_server_and_get_metadata(invalid_server_path, debug=False) + + +def test_verify_server_and_get_metadata_with_debug(valid_server_path: str, capsys) -> None: + """Test server verification with debug mode enabled.""" + server_name, server_version, required_secrets = verify_server_and_get_metadata( + valid_server_path, debug=True + ) + + # Verify returned values + assert server_name == "simpleserver" + assert server_version == "1.0.0" + assert "MY_SECRET_KEY" in required_secrets + + # Verify debug messages are printed + captured = capsys.readouterr() + assert "Started server on port" in captured.out + assert "Found 1 required secret(s)" in captured.out + assert "Server stopped" in captured.out diff --git a/libs/tests/cli/deploy/test_servers/invalid_server/pyproject.toml b/libs/tests/cli/deploy/test_servers/invalid_server/pyproject.toml new file mode 100644 index 000000000..0656d926a --- /dev/null +++ b/libs/tests/cli/deploy/test_servers/invalid_server/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "simple_server" +version = "0.1.0" +description = "MCP Server created with Arcade.dev" +requires-python = ">=3.10" +dependencies = [ + "arcade-mcp-server>=1.0.1,<2.0.0", +] + +[project.optional-dependencies] +dev = [ + "arcade-mcp[all]>=1.0.1,<2.0.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "mypy>=1.0.0", + "ruff>=0.1.0", +] + +# Tell Arcade.dev that this package has Arcade tools +[project.entry-points.arcade_toolkits] +toolkit_name = "simple_server" + +[tool.setuptools.packages.find] +include = ["simple_server*"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.mypy] +python_version = "3.10" +warn_unused_configs = true +disallow_untyped_defs = false diff --git a/libs/tests/cli/deploy/test_servers/invalid_server/server.py b/libs/tests/cli/deploy/test_servers/invalid_server/server.py new file mode 100644 index 000000000..184b7851e --- /dev/null +++ b/libs/tests/cli/deploy/test_servers/invalid_server/server.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""simple_server MCP server""" + +import sys +from typing import Annotated + +import httpx +from arcade_mcp_server import Context, MCPApp +from arcade_mcp_server.auth import Reddit + +app = MCPApp(name="simpleserver", version="1.0.0", log_level="DEBUG") + + +@app.tool +def greet(name: dict) -> str: + """Greet a person by name.""" + return f"Hello, {name}!" + + +# To use this tool, you need to either set the secret in the .env file or as an environment variable +@app.tool(requires_secrets=["MY_SECRET_KEY"]) +def whisper_secret(context: Context) -> Annotated[str, "The last 4 characters of the secret"]: + """Reveal the last 4 characters of a secret""" + # Secrets are injected into the context at runtime. + # LLMs and MCP clients cannot see or access your secrets + # You can define secrets in a .env file. + try: + secret = context.get_secret("MY_SECRET_KEY") + except Exception as e: + return str(e) + + return "The last 4 characters of the secret are: " + secret[-4:] + + +# To use this tool, you need to either set your ARCADE_API_KEY as an environment variable or +# use the Arcade CLI (uv pip install arcade-mcp) and run 'arcade login' to authenticate. +@app.tool(requires_auth=Reddit(scopes=["read"])) +async def get_posts_in_subreddit( + context: Context, subreddit: Annotated[str, "The name of the subreddit"] +) -> dict: + """Get posts from a specific subreddit""" + # Normalize the subreddit name + subreddit = subreddit.lower().replace("r/", "").replace(" ", "") + + # Prepare the httpx request + # OAuth token is injected into the context at runtime. + # LLMs and MCP clients cannot see or access your OAuth tokens. + oauth_token = context.get_auth_token_or_empty() + headers = { + "Authorization": f"Bearer {oauth_token}", + "User-Agent": "simple_server-mcp-server", + } + params = {"limit": 5} + url = f"https://oauth.reddit.com/r/{subreddit}/hot" + + # Make the request + async with httpx.AsyncClient() as client: + response = await client.get(url, headers=headers, params=params) + response.raise_for_status() + + # Return the response + return response.json() + + +# Run with specific transport +if __name__ == "__main__": + # Get transport from command line argument, default to "http" + transport = sys.argv[1] if len(sys.argv) > 1 else "http" + + # Run the server + # - "http" (default): HTTPS streaming for Cursor, VS Code, etc. + # - "stdio": Standard I/O for Claude Desktop, CLI tools, etc. + app.run(transport=transport, host="127.0.0.1", port=8000) diff --git a/libs/tests/cli/deploy/test_servers/valid_server/pyproject.toml b/libs/tests/cli/deploy/test_servers/valid_server/pyproject.toml new file mode 100644 index 000000000..0656d926a --- /dev/null +++ b/libs/tests/cli/deploy/test_servers/valid_server/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "simple_server" +version = "0.1.0" +description = "MCP Server created with Arcade.dev" +requires-python = ">=3.10" +dependencies = [ + "arcade-mcp-server>=1.0.1,<2.0.0", +] + +[project.optional-dependencies] +dev = [ + "arcade-mcp[all]>=1.0.1,<2.0.0", + "pytest>=7.0.0", + "pytest-asyncio>=0.21.0", + "mypy>=1.0.0", + "ruff>=0.1.0", +] + +# Tell Arcade.dev that this package has Arcade tools +[project.entry-points.arcade_toolkits] +toolkit_name = "simple_server" + +[tool.setuptools.packages.find] +include = ["simple_server*"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.mypy] +python_version = "3.10" +warn_unused_configs = true +disallow_untyped_defs = false diff --git a/libs/tests/cli/deploy/test_servers/valid_server/server.py b/libs/tests/cli/deploy/test_servers/valid_server/server.py new file mode 100644 index 000000000..1f028aaf2 --- /dev/null +++ b/libs/tests/cli/deploy/test_servers/valid_server/server.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""simple_server MCP server""" + +import sys +from typing import Annotated + +import httpx +from arcade_mcp_server import Context, MCPApp +from arcade_mcp_server.auth import Reddit + +app = MCPApp(name="simpleserver", version="1.0.0", log_level="DEBUG") + + +@app.tool +def greet(name: Annotated[str, "The name of the person to greet"]) -> str: + """Greet a person by name.""" + return f"Hello, {name}!" + + +# To use this tool, you need to either set the secret in the .env file or as an environment variable +@app.tool(requires_secrets=["MY_SECRET_KEY"]) +def whisper_secret(context: Context) -> Annotated[str, "The last 4 characters of the secret"]: + """Reveal the last 4 characters of a secret""" + # Secrets are injected into the context at runtime. + # LLMs and MCP clients cannot see or access your secrets + # You can define secrets in a .env file. + try: + secret = context.get_secret("MY_SECRET_KEY") + except Exception as e: + return str(e) + + return "The last 4 characters of the secret are: " + secret[-4:] + + +# To use this tool, you need to either set your ARCADE_API_KEY as an environment variable or +# use the Arcade CLI (uv pip install arcade-mcp) and run 'arcade login' to authenticate. +@app.tool(requires_auth=Reddit(scopes=["read"])) +async def get_posts_in_subreddit( + context: Context, subreddit: Annotated[str, "The name of the subreddit"] +) -> dict: + """Get posts from a specific subreddit""" + # Normalize the subreddit name + subreddit = subreddit.lower().replace("r/", "").replace(" ", "") + + # Prepare the httpx request + # OAuth token is injected into the context at runtime. + # LLMs and MCP clients cannot see or access your OAuth tokens. + oauth_token = context.get_auth_token_or_empty() + headers = { + "Authorization": f"Bearer {oauth_token}", + "User-Agent": "simple_server-mcp-server", + } + params = {"limit": 5} + url = f"https://oauth.reddit.com/r/{subreddit}/hot" + + # Make the request + async with httpx.AsyncClient() as client: + response = await client.get(url, headers=headers, params=params) + response.raise_for_status() + + # Return the response + return response.json() + + +# Run with specific transport +if __name__ == "__main__": + # Get transport from command line argument, default to "http" + transport = sys.argv[1] if len(sys.argv) > 1 else "http" + + # Run the server + # - "http" (default): HTTPS streaming for Cursor, VS Code, etc. + # - "stdio": Standard I/O for Claude Desktop, CLI tools, etc. + app.run(transport=transport, host="127.0.0.1", port=8000) diff --git a/libs/tests/deployment/test_config.py b/libs/tests/deployment/test_config.py deleted file mode 100644 index 805a78acb..000000000 --- a/libs/tests/deployment/test_config.py +++ /dev/null @@ -1,231 +0,0 @@ -# Ignore hardcoded secret linting -# ruff: noqa: S105 -# ruff: noqa: S106 -import json -import os -from pathlib import Path - -import pytest -from arcade_cli.deployment import ( - Config, - Deployment, - LocalPackages, - Package, - PackageRepository, - Pypi, - Secret, - Worker, -) - - -@pytest.fixture -def test_dir(): - return Path(__file__).parent - - -def test_invalid_toml_path(test_dir): - with pytest.raises(FileNotFoundError): - Deployment.from_toml(test_dir / "test_files" / "invalid.toml") - - -def test_missing_fields(test_dir): - with pytest.raises(ValueError): - Deployment.from_toml(test_dir / "test_files" / "invalid.fields.worker.toml") - - -def test_deployment_parsing(test_dir): - config_path = test_dir / "test_files" / "full.worker.toml" - deployment = Deployment.from_toml(config_path) - - # Test config section - assert deployment.worker[0].config.id == "test" - assert deployment.worker[0].config.enabled is True - assert deployment.worker[0].config.timeout == 10 - assert deployment.worker[0].config.retries == 3 - assert deployment.worker[0].config.secret == Secret(value="test-secret", pattern=None) - - # Test pypi section - assert deployment.worker[0].pypi_source.packages == [Package(name="arcade-x")] - - # Test local_packages section - assert deployment.worker[0].local_source.packages == ["./mock_toolkit"] - - # Test custom_repositories section - repo = deployment.worker[0].custom_source[0] - assert repo.index == "pypi" - assert repo.index_url == "https://pypi.org/simple" - assert repo.trusted_host == "pypi.org" - assert repo.packages == [Package(name="arcade-mcp", specifier=">=1.0.0")] - - repo = deployment.worker[0].custom_source[1] - assert repo.index == "pypi2" - assert repo.index_url == "https://pypi2.org/simple" - assert repo.trusted_host == "pypi2.org" - assert repo.packages == [Package(name="arcade-slack")] - - -def test_specifier(): - from packaging.requirements import Requirement - - req = Requirement("arcade-mcp>=1.0.0") - assert req.name == "arcade-mcp" - assert req.specifier == ">=1.0.0" - - -@pytest.mark.skip(reason="This test is flaky and needs to be fixed") -def test_deployment_dict(test_dir): - config_path = test_dir / "test_files" / "full.worker.toml" - deployment = Deployment.from_toml(config_path) - expected = json.loads("""{ - "name": "test", - "secret": "test-secret", - "enabled": true, - "timeout": 10, - "retries": 3, - "wait": false, - "pypi": { - "packages": [ - { - "name": "arcade-x", - "specifier": null - } - ], - "index": "pypi", - "index_url": "https://pypi.org/simple", - "trusted_host": "pypi.org" - }, - "custom_repositories": [ - { - "packages": [ - { - "name": "arcade-mcp", - "specifier": ">=1.0.0" - } - ], - "index": "pypi", - "index_url": "https://pypi.org/simple", - "trusted_host": "pypi.org" - }, - { - "packages": [ - { - "name": "arcade-slack", - "specifier": null - } - ], - "index": "pypi2", - "index_url": "https://pypi2.org/simple", - "trusted_host": "pypi2.org" - } - ], - "local_packages": [ - { - "name": "mock_toolkit", - "content": "H4sIAOgdymcC/+2XwWuDMBTGPftXZDltMNIkJtrCOrpbL4PdSxmiKXNVIzHt6n+/OAvtNrqbMur7Xd7j5YGH5Ps+JBMyWbzEh6WKU2W8XqAdlyqlgTj17ZxRzriHDt4A7GobG/d5b5zwKSpsVqg5iwRjs6kMBJEzMYtC7nvA1VPoZPtqtc63mZ14/ek/krKrYVcp/655JtyLY4wHNHL6D5iMPCSH1H+dGtX84YBubbO5vvsn4P/g/+f+LyihPBSUSfD/sfl/EWclqZo+9B8Kcdn/eXTyf+bmTAjp9E+H1P9I/b8yWWlv8VLlub5HH9rk6Q2+A+mPhf+R/8Hv/GeQ/4Pkf/Qj/3lEpAgCOQUPGF3+V01l9LtKLLG6yAfLf07F2f9fq/+QhhTyfwhW7d2TSitrmrVfxoVCc4TPXwX298rUmS7bA0oYodhPVZ2YrLLH6bNbR8d1tNEGPZnExQn2451906Z2OyvczdBDqvaL+Ksnrn3EazAaAAAAAAAAAAAAAAAAAOiJT7MTVu0AKAAA" - } - ] -}""") - got = deployment.worker[0].request().model_dump(mode="json") - # Remove encoding part that contains the content - got["local_packages"][0].pop("content") - expected["local_packages"][0].pop("content") - - assert got == expected - - -def test_missing_local_package(test_dir): - config_path = test_dir / "test_files" / "invalid.localfile.worker.toml" - deployment = Deployment.from_toml(config_path) - with pytest.raises(FileNotFoundError): - deployment.worker[0].request() - - -def test_invalid_local_package(test_dir): - config_path = test_dir / "test_files" / "invalid.localfile.worker.toml" - deployment = Deployment.from_toml(config_path) - with pytest.raises(FileNotFoundError): - deployment.worker[1].request() - - -def test_unconfigured_local_package(test_dir): - config_path = test_dir / "test_files" / "invalid.localfile.worker.toml" - deployment = Deployment.from_toml(config_path) - with pytest.raises(ValueError): - deployment.worker[2].request() - - -def test_duplicate_pypi_packages(): - worker = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - pypi_source=Pypi(packages=["arcade-slack", "arcade-slack"]), - ) - with pytest.raises(ValueError): - worker.validate_packages() - - -def test_duplicate_custom_repository_packages(): - worker = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - custom_source=[ - PackageRepository( - index="pypi", - index_url="https://pypi.org/simple", - trusted_host="pypi.org", - packages=["arcade-slack", "arcade-slack"], - ) - ], - ) - with pytest.raises(ValueError): - worker.validate_packages() - - -def test_duplicate_local_packages(): - worker = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - local_source=LocalPackages(packages=["./mock_toolkit", "./mock_toolkit"]), - ) - with pytest.raises(ValueError): - worker.validate_packages() - - -def test_duplicate_all_typed_packages(): - worker = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - pypi_source=Pypi(packages=["arcade-slack"]), - custom_source=[ - PackageRepository( - index="pypi", - index_url="https://pypi.org/simple", - trusted_host="pypi.org", - packages=["arcade-slack", "arcade-x"], - ) - ], - local_source=LocalPackages(packages=["./arcade-x"]), - ) - with pytest.raises(ValueError): - worker.validate_packages() - - -def test_duplicate_worker_names(): - worker = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - ) - worker2 = Worker( - toml_path=Path(__file__), - config=Config(id="test", secret=Secret(value="test-secret", pattern=None)), - ) - with pytest.raises(ValueError): - Deployment(workers=[worker, worker2]) - - -def test_secret_parsing(test_dir): - os.environ["TEST_WORKER_SECRET"] = "test-secret" - deployment = Deployment.from_toml(test_dir / "test_files" / "env.secret.worker.toml") - assert deployment.worker[0].config.secret == Secret( - value="test-secret", pattern="TEST_WORKER_SECRET" - ) diff --git a/libs/tests/deployment/test_files/env.secret.worker.toml b/libs/tests/deployment/test_files/env.secret.worker.toml deleted file mode 100644 index 309f61af2..000000000 --- a/libs/tests/deployment/test_files/env.secret.worker.toml +++ /dev/null @@ -1,5 +0,0 @@ - -[[worker]] -[worker.config] -id = "test" -secret = "${env: TEST_WORKER_SECRET}" diff --git a/libs/tests/deployment/test_files/full.worker.toml b/libs/tests/deployment/test_files/full.worker.toml deleted file mode 100644 index 3b03b3d66..000000000 --- a/libs/tests/deployment/test_files/full.worker.toml +++ /dev/null @@ -1,26 +0,0 @@ - -[[worker]] -[worker.config] -id = "test" -enabled = true -timeout = 10 -retries = 3 -secret = "test-secret" - -[worker.pypi_source] -packages = ["arcade-x"] - -[worker.local_source] -packages = ["./mock_toolkit"] - -[[worker.custom_source]] -index = "pypi" -index_url = "https://pypi.org/simple" -trusted_host = "pypi.org" -packages = ["arcade-mcp>=1.0.0"] - -[[worker.custom_source]] -index = "pypi2" -index_url = "https://pypi2.org/simple" -trusted_host = "pypi2.org" -packages = ["arcade-slack"] diff --git a/libs/tests/deployment/test_files/invalid.fields.worker.toml b/libs/tests/deployment/test_files/invalid.fields.worker.toml deleted file mode 100644 index bdecbcc0c..000000000 --- a/libs/tests/deployment/test_files/invalid.fields.worker.toml +++ /dev/null @@ -1,3 +0,0 @@ - -[[worker]] -[worker.config] diff --git a/libs/tests/deployment/test_files/invalid.localfile.worker.toml b/libs/tests/deployment/test_files/invalid.localfile.worker.toml deleted file mode 100644 index 9ca398fb1..000000000 --- a/libs/tests/deployment/test_files/invalid.localfile.worker.toml +++ /dev/null @@ -1,42 +0,0 @@ - -[[worker]] -[worker.config] -id = "test" -enabled = true -timeout = 10 -retries = 3 -secret = "test-secret" - -[worker.pypi_source] -packages = ["arcade-mcp"] - -[worker.local_source] -packages = ["./missing_toolkit"] - -[[worker]] -[worker.config] -id = "test-2" -enabled = true -timeout = 10 -retries = 3 -secret = "test-secret" - -[worker.pypi_source] -packages = ["arcade-mcp"] - -[worker.local_source] -packages = ["./invalid.localfile.worker.toml"] - -[[worker]] -[worker.config] -id = "test-3" -enabled = true -timeout = 10 -retries = 3 -secret = "test-secret" - -[worker.pypi_source] -packages = ["arcade-mcp"] - -[worker.local_source] -packages = ["./invalid_toolkit"] diff --git a/libs/tests/deployment/test_files/invalid.secret.worker.toml b/libs/tests/deployment/test_files/invalid.secret.worker.toml deleted file mode 100644 index 4b6697c47..000000000 --- a/libs/tests/deployment/test_files/invalid.secret.worker.toml +++ /dev/null @@ -1,7 +0,0 @@ -[[worker]] -[worker.config] -id = "test" -enabled = true -timeout = 10 -retries = 3 -secret = "dev" diff --git a/libs/tests/deployment/test_files/invalid_toolkit/invalid_main.py b/libs/tests/deployment/test_files/invalid_toolkit/invalid_main.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/libs/tests/deployment/test_files/mock_toolkit/mock_main.py b/libs/tests/deployment/test_files/mock_toolkit/mock_main.py deleted file mode 100644 index f7cf60e14..000000000 --- a/libs/tests/deployment/test_files/mock_toolkit/mock_main.py +++ /dev/null @@ -1 +0,0 @@ -print("Hello, world!") diff --git a/libs/tests/deployment/test_files/mock_toolkit/pyproject.toml b/libs/tests/deployment/test_files/mock_toolkit/pyproject.toml deleted file mode 100644 index db164e63c..000000000 --- a/libs/tests/deployment/test_files/mock_toolkit/pyproject.toml +++ /dev/null @@ -1,5 +0,0 @@ -[tool.poetry] -name = "mock_toolkit" -version = "0.1.0" -description = "Mock toolkit for Arcade" -authors = ["Arcade "] From 71a0bf4de4f428d8580002afebf99cef835e4673 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Sun, 12 Oct 2025 20:39:18 -0700 Subject: [PATCH 5/7] Get rid of super long running tests --- libs/tests/cli/deploy/test_deploy.py | 74 ++-------------------------- 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/libs/tests/cli/deploy/test_deploy.py b/libs/tests/cli/deploy/test_deploy.py index 5f8619b56..0a6768bc6 100644 --- a/libs/tests/cli/deploy/test_deploy.py +++ b/libs/tests/cli/deploy/test_deploy.py @@ -143,35 +143,6 @@ def test_create_package_archive_excludes_files(tmp_path: Path) -> None: assert any("pyproject.toml" in name for name in filenames) -# Tests for start_server_process - - -def test_start_server_process_success(valid_server_path: str) -> None: - """Test starting a valid server process.""" - process, port = start_server_process(valid_server_path, debug=False) - - try: - # Verify process is running - assert process.poll() is None, "Process should be running" - - # Verify port is in expected range - assert 8000 <= port <= 9000 - - # Give the process a moment to start - time.sleep(1) - - # Verify process is still running - assert process.poll() is None, "Process should still be running" - finally: - # Clean up - process.terminate() - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - process.kill() - process.wait() - - # Tests for wait_for_health @@ -181,12 +152,7 @@ def test_wait_for_health_success(valid_server_path: str, capsys) -> None: base_url = f"http://127.0.0.1:{port}" try: - # This should succeed without raising wait_for_health(base_url, process, timeout=10) - - # Verify success message was printed - captured = capsys.readouterr() - assert "Server is healthy" in captured.out finally: # Clean up process.terminate() @@ -207,7 +173,7 @@ def test_wait_for_health_process_dies(valid_server_path: str) -> None: process.wait() # Mock process object to pass to wait_for_health - with pytest.raises(ValueError, match="Server failed to become healthy"): + with pytest.raises(ValueError): wait_for_health(base_url, process, timeout=2) @@ -223,16 +189,10 @@ def test_get_server_info_success(valid_server_path: str, capsys) -> None: # Wait for server to be healthy first wait_for_health(base_url, process, timeout=10) - # Get server info server_name, server_version = get_server_info(base_url) - # Verify expected values assert server_name == "simpleserver" assert server_version == "1.0.0" - - # Verify success message was printed - captured = capsys.readouterr() - assert "Found server: simpleserver v1.0.0" in captured.out finally: # Clean up process.terminate() @@ -247,7 +207,7 @@ def test_get_server_info_invalid_url() -> None: """Test that invalid URL raises ValueError.""" invalid_url = "http://127.0.0.1:9999" - with pytest.raises(ValueError, match="Failed to extract server info from /mcp endpoint"): + with pytest.raises(ValueError): get_server_info(invalid_url) @@ -263,16 +223,8 @@ def test_get_required_secrets_with_secrets(valid_server_path: str, capsys) -> No # Wait for server to be healthy first wait_for_health(base_url, process, timeout=10) - # Get required secrets secrets = get_required_secrets(base_url, "simpleserver", "1.0.0", debug=True) - - # Verify expected secrets assert "MY_SECRET_KEY" in secrets - - # Verify console output - captured = capsys.readouterr() - assert "simpleserver v1.0.0 has 3 tools" in captured.out - assert "Found 1 required secret(s)" in captured.out finally: # Clean up process.terminate() @@ -292,11 +244,9 @@ def test_get_required_secrets_no_secrets(valid_server_path: str) -> None: # Wait for server to be healthy first wait_for_health(base_url, process, timeout=10) - # Get required secrets (even though server has secrets, function should work) secrets = get_required_secrets(base_url, "simpleserver", "1.0.0", debug=False) - # Verify it returns a set - assert isinstance(secrets, set) + assert len(secrets) == 1 finally: # Clean up process.terminate() @@ -331,18 +281,6 @@ def test_verify_server_and_get_metadata_success(valid_server_path: str, capsys) assert server_version == "1.0.0" assert "MY_SECRET_KEY" in required_secrets - # Verify console output - captured = capsys.readouterr() - assert "Verifying server and extracting metadata" in captured.out - assert "Server is healthy" in captured.out - assert "Found server: simpleserver v1.0.0" in captured.out - - -def test_verify_server_and_get_metadata_invalid_server(invalid_server_path: str) -> None: - """Test that invalid server raises ValueError with cleanup.""" - with pytest.raises(ValueError, match="Server process exited immediately"): - verify_server_and_get_metadata(invalid_server_path, debug=False) - def test_verify_server_and_get_metadata_with_debug(valid_server_path: str, capsys) -> None: """Test server verification with debug mode enabled.""" @@ -354,9 +292,3 @@ def test_verify_server_and_get_metadata_with_debug(valid_server_path: str, capsy assert server_name == "simpleserver" assert server_version == "1.0.0" assert "MY_SECRET_KEY" in required_secrets - - # Verify debug messages are printed - captured = capsys.readouterr() - assert "Started server on port" in captured.out - assert "Found 1 required secret(s)" in captured.out - assert "Server stopped" in captured.out From 6f7762f5ac05cae869ae0dc5dd31d063b371c812 Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Mon, 13 Oct 2025 17:37:29 -0700 Subject: [PATCH 6/7] Better logs --- libs/arcade-cli/arcade_cli/deploy.py | 84 ++++++++++--------- .../arcade_mcp_server/mcp_app.py | 21 ++++- libs/tests/arcade_mcp_server/test_mcp_app.py | 53 ++++++++++-- 3 files changed, 106 insertions(+), 52 deletions(-) diff --git a/libs/arcade-cli/arcade_cli/deploy.py b/libs/arcade-cli/arcade_cli/deploy.py index c60731fc6..eb97b2a91 100644 --- a/libs/arcade-cli/arcade_cli/deploy.py +++ b/libs/arcade-cli/arcade_cli/deploy.py @@ -169,9 +169,6 @@ def start_server_process(entrypoint: str, debug: bool = False) -> tuple[subproce env=env, ) - if debug: - console.print(f" Started server on port {port}", style="dim") - # Check for immediate failure on start up time.sleep(0.5) if process.poll() is not None: @@ -261,7 +258,8 @@ def get_server_info(base_url: str) -> tuple[str, str]: server_name = mcp_data["result"]["serverInfo"]["name"] server_version = mcp_data["result"]["serverInfo"]["version"] - console.print(f"✓ Found server: {server_name} v{server_version}", style="green") + console.print(f"✓ Found server name: {server_name}", style="green") + console.print(f"✓ Found server version: {server_version}", style="green") except Exception as e: raise ValueError(f"Failed to extract server info from /mcp endpoint: {e}") from e @@ -306,12 +304,7 @@ def get_required_secrets( if secret.get("key"): required_secrets.add(secret["key"]) - console.print(f"{server_name} v{server_version} has {len(tools_data)} tools", style="dim") - if debug: - if required_secrets: - console.print(f" Found {len(required_secrets)} required secret(s)", style="dim") - else: - console.print(" No secrets required", style="dim") + console.print(f"✓ Found {len(tools_data)} tools", style="green") except Exception as e: raise ValueError(f"Failed to extract tool secrets from /worker/tools endpoint: {e}") from e @@ -343,15 +336,17 @@ def verify_server_and_get_metadata( Raises: ValueError: If the server fails to start or metadata extraction fails """ - console.print("\nVerifying server and extracting metadata...", style="dim") - process, port = start_server_process(entrypoint, debug) + console.print(f"✓ Server started on port {port}", style="green") base_url = f"http://127.0.0.1:{port}" try: wait_for_health(base_url, process) + server_name, server_version = get_server_info(base_url) + required_secrets = get_required_secrets(base_url, server_name, server_version, debug) + console.print(f"✓ Found {len(required_secrets)} required secret(s)", style="green") return server_name, server_version, required_secrets @@ -365,7 +360,7 @@ def verify_server_and_get_metadata( process.wait() if debug: - console.print(" Server stopped", style="dim") + console.print("✓ Server stopped", style="green") def upsert_secrets_to_engine( @@ -383,27 +378,19 @@ def upsert_secrets_to_engine( if not secrets: return - console.print(f"\nRequired secrets: {', '.join(sorted(secrets))}", style="dim") - client = httpx.Client(base_url=engine_url, headers={"Authorization": f"Bearer {api_key}"}) for secret_key in sorted(secrets): secret_value = os.getenv(secret_key) - if debug: - if secret_value: - console.print( - f" Found secret '{secret_key}' in environment (value ends with ...{secret_value[-4:]})", - style="dim", - ) - else: - console.print( - f" Secret '{secret_key}' not found in environment", style="dim yellow" - ) - - if not secret_value: + if secret_value: console.print( - f"⚠️ Secret '{secret_key}' not found in environment, skipping.", + f"✓ Uploading '{secret_key}' with value ending in ...{secret_value[-4:]}", + style="green", + ) + else: + console.print( + f"⚠️ Secret '{secret_key}' not found in environment, skipping upload.", style="yellow", ) continue @@ -416,7 +403,7 @@ def upsert_secrets_to_engine( timeout=30, ) response.raise_for_status() - console.print(f"✓ Secret '{secret_key}' uploaded", style="dim green") + console.print(f"✓ Secret '{secret_key}' uploaded", style="green") except httpx.HTTPStatusError as e: error_msg = f"Failed to upload secret '{secret_key}': HTTP {e.response.status_code}" if debug: @@ -496,27 +483,34 @@ def deploy_server_logic( debug: Show debug information """ # Step 1: Validate user is logged in + console.print("\nValidating user is logged in...", style="dim") config = validate_and_get_config() engine_url = compute_base_url(force_tls, force_no_tls, host, port) + console.print(f"✓ {config.user.email} is logged in", style="green") # Step 2: Validate pyproject.toml exists in current directory + console.print("\nValidating pyproject.toml exists in current directory...", style="dim") current_dir = Path.cwd() pyproject_path = current_dir / "pyproject.toml" if not pyproject_path.exists(): raise FileNotFoundError( - f"pyproject.toml not found in current directory: {current_dir}\n" + f"pyproject.toml not found at {pyproject_path}\n" "Please run this command from the root of your MCP server package." ) + console.print(f"✓ pyproject.toml found at {pyproject_path}", style="green") # Step 3: Load .env file from current directory if it exists + console.print("\nLoading .env file from current directory if it exists...", style="dim") env_path = current_dir / ".env" if env_path.exists(): load_dotenv(env_path, override=False) - if debug: - console.print(f" Loaded environment from {env_path}", style="dim") + console.print(f"✓ Loaded environment from {env_path}", style="green") + else: + console.print(f"⚠️ No .env file found at {env_path}", style="yellow") # Step 4: Verify server and extract metadata + console.print("\nVerifying server and extracting metadata...", style="dim") try: server_name, server_version, required_secrets = verify_server_and_get_metadata( entrypoint, debug=debug @@ -529,10 +523,10 @@ def deploy_server_logic( # Step 5: Upsert secrets to engine if required_secrets: - console.print(f"\nDiscovered {len(required_secrets)} required secret(s)", style="dim") + console.print( + f"\nUploading {len(required_secrets)} required secret(s) to Arcade...", style="dim" + ) upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) - else: - console.print("\nNo secrets required", style="dim") # Step 6: Create tar.gz archive of current directory console.print("\nCreating deployment package...", style="dim") @@ -572,11 +566,21 @@ def deploy_server_logic( console.print( f"✓ Server '{server_name}' v{server_version} deployed successfully", style="bold green" ) + + deployment_id = response.get("id", "N/A") + deployment_uri = response.get("http", {}).get("uri", "N/A") + deployment_secret = response.get("http", {}).get("secret", "N/A").get("value", "N/A") + + console.print("\n[bold]Deployment Details:[/bold]") + console.print(f" • Server ID: [cyan]{deployment_id}[/cyan]") + console.print(f" • Server URI: [cyan]{deployment_uri}[/cyan]") + console.print(f" • Server Secret: [cyan]{deployment_secret}[/cyan]") + console.print("\n[yellow]⚠ Note:[/yellow] Your server is now starting up...", style="bold") console.print( - "\nView your deployed servers in the Arcade Dashboard: https://api.arcade.dev/dashboard/", - style="dim", + "\n This process may take a few minutes. Your server will be available at the URI above once ready." ) - if debug and response: - console.print("\nDeployment response:", style="dim") - console.print(response) + console.print( + "\nView and manage your servers: [link]https://api.arcade.dev/dashboard/[/link]", + style="dim", + ) diff --git a/libs/arcade-mcp-server/arcade_mcp_server/mcp_app.py b/libs/arcade-mcp-server/arcade_mcp_server/mcp_app.py index 4af3a8a83..3bccbf6b8 100644 --- a/libs/arcade-mcp-server/arcade_mcp_server/mcp_app.py +++ b/libs/arcade-mcp-server/arcade_mcp_server/mcp_app.py @@ -224,7 +224,9 @@ def run( logger.error("No tools added to the server. Use @app.tool decorator or app.add_tool().") sys.exit(1) - host, port, transport = MCPApp._get_configuration_overrides(host, port, transport) + host, port, transport, reload = MCPApp._get_configuration_overrides( + host, port, transport, reload + ) # Since the transport could have changed since __init__, we need to setup logging again self._setup_logging(transport == "stdio") @@ -257,8 +259,8 @@ def run( @staticmethod def _get_configuration_overrides( - host: str, port: int, transport: TransportType - ) -> tuple[str, int, TransportType]: + host: str, port: int, transport: TransportType, reload: bool + ) -> tuple[str, int, TransportType, bool]: """Get configuration overrides from environment variables.""" if envvar_transport := os.getenv("ARCADE_SERVER_TRANSPORT"): transport = envvar_transport @@ -284,7 +286,18 @@ def _get_configuration_overrides( f"Using '{port}' as port from ARCADE_SERVER_PORT environment variable" ) - return host, port, transport + if envvar_reload := os.getenv("ARCADE_SERVER_RELOAD"): + if envvar_reload.lower() not in ["0", "1"]: + logger.warning( + f"Invalid reload: '{envvar_reload}' from ARCADE_SERVER_RELOAD environment variable. Using default reload {reload}" + ) + else: + reload = bool(int(envvar_reload)) + logger.debug( + f"Using '{reload}' as reload from ARCADE_SERVER_RELOAD environment variable" + ) + + return host, port, transport, reload class _ToolsAPI: diff --git a/libs/tests/arcade_mcp_server/test_mcp_app.py b/libs/tests/arcade_mcp_server/test_mcp_app.py index def09c0b7..a88980dcf 100644 --- a/libs/tests/arcade_mcp_server/test_mcp_app.py +++ b/libs/tests/arcade_mcp_server/test_mcp_app.py @@ -226,23 +226,31 @@ def test_get_configuration_overrides(self, monkeypatch): monkeypatch.delenv("ARCADE_SERVER_TRANSPORT", raising=False) monkeypatch.delenv("ARCADE_SERVER_HOST", raising=False) monkeypatch.delenv("ARCADE_SERVER_PORT", raising=False) + monkeypatch.delenv("ARCADE_SERVER_RELOAD", raising=False) # Test default values (no environment variables) - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) assert host == "127.0.0.1" assert port == 8000 assert transport == "http" + assert not reload # Test transport override monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "stdio") - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) assert transport == "stdio" monkeypatch.delenv("ARCADE_SERVER_TRANSPORT") # Test host override (only works with HTTP transport) monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "http") monkeypatch.setenv("ARCADE_SERVER_HOST", "192.168.1.1") - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) assert host == "192.168.1.1" assert transport == "http" monkeypatch.delenv("ARCADE_SERVER_HOST") @@ -250,27 +258,56 @@ def test_get_configuration_overrides(self, monkeypatch): # Test port override (only works with HTTP transport) monkeypatch.setenv("ARCADE_SERVER_PORT", "9000") - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) assert port == 9000 monkeypatch.delenv("ARCADE_SERVER_PORT") # Test invalid port value monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "http") monkeypatch.setenv("ARCADE_SERVER_PORT", "invalid_port") - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) assert port == 8000 # Should keep the default value monkeypatch.delenv("ARCADE_SERVER_PORT") monkeypatch.delenv("ARCADE_SERVER_TRANSPORT") - # Test host/port with stdio transport + # Test valid reload value + monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "http") + monkeypatch.setenv("ARCADE_SERVER_RELOAD", "1") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) + assert reload + monkeypatch.delenv("ARCADE_SERVER_RELOAD") + monkeypatch.delenv("ARCADE_SERVER_TRANSPORT") + + # Test invalid reload value + monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "http") + monkeypatch.setenv("ARCADE_SERVER_RELOAD", "invalid_reload") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) + assert not reload # Should keep the default value + monkeypatch.delenv("ARCADE_SERVER_RELOAD") + monkeypatch.delenv("ARCADE_SERVER_TRANSPORT") + + # Test host/port/reload with stdio transport monkeypatch.setenv("ARCADE_SERVER_TRANSPORT", "stdio") monkeypatch.setenv("ARCADE_SERVER_HOST", "192.168.1.1") monkeypatch.setenv("ARCADE_SERVER_PORT", "9000") - host, port, transport = MCPApp._get_configuration_overrides("127.0.0.1", 8000, "http") - # For stdio, host and port are still returned but not used by the server + monkeypatch.setenv("ARCADE_SERVER_RELOAD", "true") + host, port, transport, reload = MCPApp._get_configuration_overrides( + "127.0.0.1", 8000, "http", False + ) + # For stdio, host, port, and reload are still returned but not used by the server assert host == "127.0.0.1" # Host should remain unchanged for stdio transport assert port == 8000 # Port should remain unchanged for stdio transport assert transport == "stdio" + assert not reload + monkeypatch.delenv("ARCADE_SERVER_RELOAD") monkeypatch.delenv("ARCADE_SERVER_HOST") monkeypatch.delenv("ARCADE_SERVER_PORT") monkeypatch.delenv("ARCADE_SERVER_TRANSPORT") From 38559e1c3ac69f7980dd8fa845c74ed0dcac2d2a Mon Sep 17 00:00:00 2001 From: Eric Gustin Date: Tue, 14 Oct 2025 13:18:22 -0700 Subject: [PATCH 7/7] Add YOLO mode --- libs/arcade-cli/arcade_cli/deploy.py | 79 ++++++++++++++++++++++------ libs/arcade-cli/arcade_cli/main.py | 60 ++++++++++++++++++++- 2 files changed, 120 insertions(+), 19 deletions(-) diff --git a/libs/arcade-cli/arcade_cli/deploy.py b/libs/arcade-cli/arcade_cli/deploy.py index eb97b2a91..996b2a67a 100644 --- a/libs/arcade-cli/arcade_cli/deploy.py +++ b/libs/arcade-cli/arcade_cli/deploy.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, Field from rich.console import Console +from arcade_cli.secret import load_env_file from arcade_cli.utils import compute_base_url, validate_and_get_config console = Console() @@ -465,6 +466,10 @@ def deploy_server_to_engine( def deploy_server_logic( entrypoint: str, + skip_validate: bool, + server_name: str | None, + server_version: str | None, + secrets: str, host: str, port: int | None, force_tls: bool, @@ -475,7 +480,12 @@ def deploy_server_logic( Main logic for deploying an MCP server to Arcade Engine. Args: - entrypoint: Path to the entrypoint file containing MCPApp + entrypoint: Path (relative to project root) to the entrypoint file that runs the MCPApp instance. + This file must execute the `run()` method on your `MCPApp` instance when invoked directly. + skip_validate: Skip running the server locally for health/metadata checks. + server_name: Explicit server name to use when --skip-validate is set. + server_version: Explicit server version to use when --skip-validate is set. + secrets: How to upsert secrets before deploy. host: Arcade Engine host port: Arcade Engine port (optional) force_tls: Force TLS connection @@ -509,24 +519,59 @@ def deploy_server_logic( else: console.print(f"⚠️ No .env file found at {env_path}", style="yellow") - # Step 4: Verify server and extract metadata - console.print("\nVerifying server and extracting metadata...", style="dim") - try: - server_name, server_version, required_secrets = verify_server_and_get_metadata( - entrypoint, debug=debug - ) - except Exception as e: - raise ValueError( - f"Server verification failed: {e}\n" - "Please ensure your server starts correctly before deploying." - ) from e - - # Step 5: Upsert secrets to engine - if required_secrets: + # Step 4: Verify server and extract metadata (or skip if --skip-validate) + required_secrets_from_validation: set[str] = set() + + if skip_validate: + console.print("\n⚠️ Skipping server validation (--skip-validate set)", style="yellow") + # Use the provided server_name and server_version + # These are guaranteed to be set due to validation in main.py + if server_name is None: + raise ValueError("server_name must be provided when skip_validate is True") + if server_version is None: + raise ValueError("server_version must be provided when skip_validate is True") + console.print(f"✓ Using server name: {server_name}", style="green") + console.print(f"✓ Using server version: {server_version}", style="green") + else: console.print( - f"\nUploading {len(required_secrets)} required secret(s) to Arcade...", style="dim" + "\nValidating server is healthy and extracting metadata before deploying...", + style="dim", ) - upsert_secrets_to_engine(engine_url, config.api.key, required_secrets, debug) + try: + server_name, server_version, required_secrets_from_validation = ( + verify_server_and_get_metadata(entrypoint, debug=debug) + ) + except Exception as e: + raise ValueError( + f"Server verification failed: {e}\n" + "Please ensure your server starts correctly before deploying." + ) from e + + # Step 5: Determine which secrets to upsert based on --secrets flag + secrets_to_upsert: set[str] = set() + + if secrets == "skip": + console.print("\n⚠️ Skipping secret upload (--secrets skip)", style="yellow") + elif secrets == "all": + console.print("\nUploading ALL secrets from .env file...", style="dim") + secrets_to_upsert = set(load_env_file(str(env_path)).keys()) + if secrets_to_upsert: + console.print(f"✓ Found {len(secrets_to_upsert)} secret(s) in .env file", style="green") + upsert_secrets_to_engine(engine_url, config.api.key, secrets_to_upsert, debug) + else: + console.print("⚠️ No secrets found in .env file", style="yellow") + elif secrets == "auto": + # Only upload required secrets discovered during validation + if required_secrets_from_validation: + console.print( + f"\nUploading {len(required_secrets_from_validation)} required secret(s) to Arcade...", + style="dim", + ) + upsert_secrets_to_engine( + engine_url, config.api.key, required_secrets_from_validation, debug + ) + else: + console.print("\n✓ No required secrets found", style="green") # Step 6: Create tar.gz archive of current directory console.print("\nCreating deployment package...", style="dim") diff --git a/libs/arcade-cli/arcade_cli/main.py b/libs/arcade-cli/arcade_cli/main.py index f9618dd55..2e6613e40 100644 --- a/libs/arcade-cli/arcade_cli/main.py +++ b/libs/arcade-cli/arcade_cli/main.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Optional +import click import typer from arcadepy import Arcade from rich.console import Console @@ -529,13 +530,55 @@ def configure( handle_cli_error(f"Failed to configure {client}", e, debug) -@cli.command(name="deploy", help="Deploy MCP servers to Arcade", rich_help_panel="Run") +@cli.command( + name="deploy", + help="Deploy MCP servers to Arcade", + rich_help_panel="Run", +) def deploy( entrypoint: str = typer.Option( "server.py", "--entrypoint", "-e", - help="Relative path to the Python file that runs the MCPApp instance (relative to project root)", + help="Relative path to the Python file that runs the MCPApp instance (relative to project root). This file must execute the `run()` method on your `MCPApp` instance when invoked directly.", + ), + skip_validate: bool = typer.Option( + False, + "--skip-validate", + "--yolo", + help="Skip running the server locally for health/metadata checks. " + "When set, you must provide `--server-name` and `--server-version`. " + "Secret handling is controlled by `--secrets`.", + rich_help_panel="Advanced", + ), + server_name: Optional[str] = typer.Option( + None, + "--server-name", + "-n", + help="Explicit server name to use when `--skip-validate` is set. Only used when `--skip-validate` is set.", + rich_help_panel="Advanced", + ), + server_version: Optional[str] = typer.Option( + None, + "--server-version", + "-v", + help="Explicit server version to use when `--skip-validate` is set. Only used when `--skip-validate` is set.", + rich_help_panel="Advanced", + ), + secrets: str = typer.Option( + "auto", + "--secrets", + "-s", + help=( + "How to upsert secrets before deploy:\n" + " `auto` (default): During validation, discover required secret KEYS and upsert only those. " + "If `--skip-validate` is set, `auto` becomes `skip`.\n" + " `all`: Upsert every key/value pair from your server's .env file regardless of what the server needs.\n" + " `skip`: Do not upsert any secrets (assumes they are already present in Arcade)." + ), + show_choices=True, + rich_help_panel="Advanced", + click_type=click.Choice(["auto", "all", "skip"], case_sensitive=False), ), host: str = typer.Option( PROD_ENGINE_HOST, @@ -575,12 +618,25 @@ def deploy( cd my_mcp_server/ arcade deploy arcade deploy --entrypoint src/server.py + arcade deploy --skip-validate --server-name my_server_name --server-version 1.0.0 """ from arcade_cli.deploy import deploy_server_logic + if skip_validate and not (server_name and server_version): + handle_cli_error( + "When --skip-validate is set, you must provide --server-name and --server-version.", + should_exit=True, + ) + if skip_validate and secrets == "auto": + secrets = "skip" + try: deploy_server_logic( entrypoint=entrypoint, + skip_validate=skip_validate, + server_name=server_name, + server_version=server_version, + secrets=secrets, host=host, port=port, force_tls=force_tls,