diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d8cd7b1..883dca22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,30 @@ -# CHANGELOG +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + ### Added -- Support for both `rpicam-vid` (Raspberry Pi OS Trixie) and `libcamera-vid` (Raspberry Pi OS Bookworm) camera commands in `src/ac_training_lab/picam/device.py` to ensure compatibility across different OS versions. +- BO / Prefect HiTL Slack integration tutorial (2025-12-01) + - Added `scripts/prefect_scripts/bo_hitl_slack_tutorial.py` - Bayesian Optimization with human-in-the-loop evaluation via Slack + - Uses Ax Service API for Bayesian optimization + - Integrates Prefect interactive workflows with pause_flow_run + - Slack notifications for experiment suggestions + - MongoDB Atlas storage for experiment data + - Evaluation via HuggingFace Branin space + +### Changed +- Support for both `rpicam-vid` (Raspberry Pi OS Trixie) and `libcamera-vid` (Raspberry Pi OS Bookworm) camera commands ### Fixed -- Ctrl+C interrupt handling in `src/ac_training_lab/picam/device.py` now properly exits the streaming loop instead of restarting. +- Ctrl+C interrupt handling in `src/ac_training_lab/picam/device.py` ## [1.1.0] - 2024-06-11 ### Added -- Imperial (10-32 thread) alternative design to SEM door automation bill of materials in `docs/sem-door-automation-components.md`. -- Validated McMaster-Carr part numbers and direct links for all imperial components. - -### Changed -- No changes to metric design section. +- Imperial (10-32 thread) alternative design to SEM door automation bill of materials ### Notes -- All components sourced from McMaster-Carr for reliability and reproducibility. +- All components sourced from McMaster-Carr for reliability and reproducibility diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 00000000..d5ec6f0d --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,9 @@ +{ + "executionEnvironments": [ + { + "root": ".", + "pythonPath": "python", + "extraPaths": ["./src"] + } + ] +} \ No newline at end of file diff --git a/scripts/prefect_scripts/README_BO_HITL_Tutorial.md b/scripts/prefect_scripts/README_BO_HITL_Tutorial.md new file mode 100644 index 00000000..3d8464af --- /dev/null +++ b/scripts/prefect_scripts/README_BO_HITL_Tutorial.md @@ -0,0 +1,70 @@ +# Bayesian Optimization Human-in-the-Loop Slack Integration Tutorial + +Demonstrates a BO campaign with human evaluation via Slack and Prefect. + +## Workflow + +1. **Run script** - starts BO campaign via Ax Service API +2. **Ax suggests parameters** - sends Slack notification with x1, x2 values +3. **User evaluates** - uses HuggingFace Branin space +4. **User resumes** - enters objective value in Prefect UI via Slack link +5. **Loop continues** - 5 iterations to find optimal parameters + +## Setup + +### 1. Install Dependencies + +```bash +pip install ax-platform prefect prefect-slack pymongo +``` + +### 2. Start Prefect Server + +```bash +prefect server start +``` + +### 3. Configure Slack Webhook Block + +```python +from prefect.blocks.notifications import SlackWebhook + +slack_webhook_block = SlackWebhook(url="YOUR_SLACK_WEBHOOK_URL") +slack_webhook_block.save("prefect-test") +``` + +Get webhook URL from https://api.slack.com/apps + +### 4. Configure MongoDB (Optional) + +Set environment variable for experiment storage: +```bash +export MONGODB_URI="mongodb+srv://user:pass@cluster.mongodb.net/" +``` + +### 5. Run + +```bash +python bo_hitl_slack_tutorial.py +``` + +## Files + +- `bo_hitl_slack_tutorial.py` - Main tutorial (single file implementation) +- `requirements.txt` - Dependencies + +## Demo Video + +Show: +1. Running script +2. Receiving Slack notification +3. Evaluating on HuggingFace Branin space +4. Clicking Slack link to Prefect UI +5. Entering objective value +6. Repeat 4-5 times + +## References + +- [Ax Documentation](https://ax.dev/) +- [Prefect Interactive Workflows](https://docs.prefect.io/latest/guides/creating-interactive-workflows/) +- [HuggingFace Branin Space](https://huggingface.co/spaces/AccelerationConsortium/branin) diff --git a/scripts/prefect_scripts/bo_hitl_slack_tutorial.py b/scripts/prefect_scripts/bo_hitl_slack_tutorial.py new file mode 100644 index 00000000..823aa39f --- /dev/null +++ b/scripts/prefect_scripts/bo_hitl_slack_tutorial.py @@ -0,0 +1,189 @@ +""" +Human-in-the-Loop Bayesian Optimization with Ax, Prefect and Slack + +Demonstrates a BO campaign where: +1. Ax suggests parameters via Service API +2. Slack notification sent with parameters +3. Human evaluates via HuggingFace Branin space +4. Human enters objective value in Prefect UI +5. Loop continues for n iterations +""" + +import os +import json +from datetime import datetime +from pymongo import MongoClient +from ax.service.ax_client import AxClient, ObjectiveProperties +from prefect import flow, get_run_logger +from prefect.blocks.notifications import SlackWebhook +from prefect.context import get_run_context +from prefect.input import RunInput +from prefect.flow_runs import pause_flow_run +from prefect.blocks.system import Secret + + +class ExperimentInput(RunInput): + """Input model for experiment evaluation""" + objective_value: float + notes: str = "" + + +@flow(name="bo-hitl-slack-campaign") +def run_bo_campaign(n_iterations: int = 5, random_seed: int = 42, slack_block_name: str = "tutorial-slack-webhook-url", mongodb_block_name: str = "tutorial-mongodb-uri"): + """ + Bayesian Optimization campaign with human-in-the-loop evaluation via Slack + + Args: + n_iterations: Number of BO iterations to run + random_seed: Seed for Ax reproducibility + slack_block_name: Name of the Prefect Slack webhook block + mongodb_block_name: Name of the Prefect Secret block containing MongoDB URI + """ + logger = get_run_logger() + logger.info(f"Starting BO campaign with {n_iterations} iterations") + + # Initialize Ax client with Service API + ax_client = AxClient(random_seed=random_seed) + ax_client.create_experiment( + name="branin_bo_experiment", + parameters=[ + {"name": "x1", "type": "range", "bounds": [-5.0, 10.0], "value_type": "float"}, + {"name": "x2", "type": "range", "bounds": [0.0, 15.0], "value_type": "float"}, + ], + objectives={"branin": ObjectiveProperties(minimize=True)} + ) + + # Load Slack webhook + slack_block = SlackWebhook.load(slack_block_name) + + # Connect to MongoDB Atlas for storage using Prefect Secret block + mongodb_secret = Secret.load(mongodb_block_name) + mongodb_uri = mongodb_secret.get() + mongo_client = MongoClient(mongodb_uri) if mongodb_uri else None + logger.info(f"Connected to MongoDB using block '{mongodb_block_name}'") + + db = mongo_client["bo_experiments"] if mongo_client else None + + # Create experiment record + experiment_id = f"exp_{datetime.utcnow().strftime('%Y%m%d_%H%M%S')}" + if db is not None: + db.experiments.insert_one({ + "experiment_id": experiment_id, + "created_at": datetime.utcnow(), + "n_iterations": n_iterations, + "random_seed": random_seed, + "status": "running" + }) + + results = [] + + for iteration in range(n_iterations): + logger.info(f"Iteration {iteration + 1}/{n_iterations}") + + # Get next suggestion from Ax + parameters, trial_index = ax_client.get_next_trial() + x1, x2 = parameters['x1'], parameters['x2'] + + logger.info(f"Suggested: x1={x1}, x2={x2}") + + # Build Prefect Cloud UI URL - use workspace ID format + flow_run = get_run_context().flow_run + # Default to generic dashboard URL that will handle authentication and routing + account_id = os.getenv("PREFECT_ACCOUNT_ID", "5b838504-64cf-4297-9b35-b881ac6169b3") + workspace_id = os.getenv("PREFECT_WORKSPACE_ID", "d2718b4c-b49a-43ce-83c2-baf6fb3b9665") + base_url = os.getenv("PREFECT_UI_URL", "https://app.prefect.cloud") + flow_run_url = f"{base_url}/account/{account_id}/workspace/{workspace_id}/flow-runs/flow-run/{flow_run.id}" if flow_run else "" + + # Send Slack notification + message = f"""*BO Iteration {iteration + 1}/{n_iterations}* + + Evaluate Branin function at: + - x1 = {x1} + - x2 = {x2} + + Use: https://huggingface.co/spaces/AccelerationConsortium/branin + + <{flow_run_url}|Click here to resume> and enter the objective value.""" + + slack_block.notify(message) + + # Pause for human input + logger.info("Waiting for human evaluation...") + user_input = pause_flow_run( + wait_for_input=ExperimentInput.with_initial_data( + description=f"Enter objective value for x1={x1}, x2={x2}" + ) + ) + + objective_value = user_input.objective_value + logger.info(f"Received: {objective_value}") + + # Complete trial in Ax + ax_client.complete_trial(trial_index=trial_index, raw_data=objective_value) + + # Store trial result + trial_result = { + "iteration": iteration + 1, + "trial_index": trial_index, + "parameters": parameters, + "objective_value": objective_value, + "notes": user_input.notes, + "timestamp": datetime.utcnow() + } + results.append(trial_result) + + # Save to MongoDB + if db is not None: + db.trials.insert_one({ + "experiment_id": experiment_id, + **trial_result + }) + + logger.info(f"Completed iteration {iteration + 1}") + + # Get best parameters + best_parameters, best_values = ax_client.get_best_parameters() + + # Update experiment status + if db is not None: + db.experiments.update_one( + {"experiment_id": experiment_id}, + {"$set": {"status": "completed", "completed_at": datetime.utcnow(), + "best_parameters": best_parameters, "best_value": best_values[0]['branin']}} + ) + + # Send completion notification + slack_block.notify(f"""*BO Campaign Completed* + +Best parameters: x1={best_parameters['x1']}, x2={best_parameters['x2']} +Best value: {best_values[0]['branin']} +Experiment ID: {experiment_id}""") + + logger.info(f"Campaign complete. Best: {best_parameters}, Value: {best_values}") + + # Export data to JSON files + if db is not None: + # Create experiment_data folder if it doesn't exist + os.makedirs('experiment_data', exist_ok=True) + + logger.info("Exporting experiments...") + experiments = list(db.experiments.find()) + with open('experiment_data/bo_experiments.json', 'w') as f: + json.dump(experiments, f, indent=2, default=str) + + logger.info("Exporting trials...") + trials = list(db.trials.find()) + with open('experiment_data/bo_trials.json', 'w') as f: + json.dump(trials, f, indent=2, default=str) + + summary = {"experiments": experiments, "trials": trials} + with open('experiment_data/bo_data_complete.json', 'w') as f: + json.dump(summary, f, indent=2, default=str) + + if mongo_client: + mongo_client.close() + + return ax_client, results, experiment_id + + +run_bo_campaign(n_iterations=5, random_seed=42) \ No newline at end of file diff --git a/scripts/prefect_scripts/client_scripts/get_result.py b/scripts/prefect_scripts/client_scripts/get_result.py deleted file mode 100644 index 54bd09c6..00000000 --- a/scripts/prefect_scripts/client_scripts/get_result.py +++ /dev/null @@ -1,20 +0,0 @@ -import asyncio - -from prefect import get_client - - -async def get_result(flow_id): - async with get_client() as client: - response = await client.hello() - print(response.json()) # 👋 - result = (await client.read_flow_run(flow_id)).state.result() - return result - - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - result = loop.run_until_complete(get_result("bd33b4ee-7bf0-48e3-9629-23bdf121c107")) - # PersistedResult(type='reference', artifact_type=None, artifact_description=None, serializer_type='pickle', storage_block_id=UUID('1e4ce198-a25a-4808-81db-65cb30d0cffb'), storage_key='69b055e353b745249d493350b79e81e8') # noqa - loop.close() - - 1 + 1 diff --git a/scripts/prefect_scripts/client_scripts/prefect_client_basic.py b/scripts/prefect_scripts/client_scripts/prefect_client_basic.py deleted file mode 100644 index ae641c01..00000000 --- a/scripts/prefect_scripts/client_scripts/prefect_client_basic.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio - -from prefect import get_client - - -async def hello(): - async with get_client() as client: - response = await client.hello() - print(response.json()) # 👋 - - -if __name__ == "__main__": - loop = asyncio.get_event_loop() - loop.run_until_complete(hello()) - loop.close() - - 1 + 1 diff --git a/scripts/prefect_scripts/experiment_data/bo_data_complete.json b/scripts/prefect_scripts/experiment_data/bo_data_complete.json new file mode 100644 index 00000000..4c2e4f8b --- /dev/null +++ b/scripts/prefect_scripts/experiment_data/bo_data_complete.json @@ -0,0 +1,85 @@ +{ + "experiments": [ + { + "_id": "6933a24540ff909521ebc0c0", + "experiment_id": "exp_20251206_032557", + "created_at": "2025-12-06 03:25:57.991000", + "n_iterations": 5, + "random_seed": 42, + "status": "completed", + "best_parameters": { + "x1": -3.3329896349459887, + "x2": 11.819649185054004 + }, + "best_value": 1.4197303445112155, + "completed_at": "2025-12-06 03:31:54.966000" + } + ], + "trials": [ + { + "_id": "6933a2fc40ff909521ebc0c1", + "experiment_id": "exp_20251206_032557", + "iteration": 1, + "trial_index": 0, + "parameters": { + "x1": 9.962700307369232, + "x2": 1.565495878458023 + }, + "objective_value": 3.715720823042817, + "notes": "", + "timestamp": "2025-12-06 03:29:00.946000" + }, + { + "_id": "6933a32640ff909521ebc0c2", + "experiment_id": "exp_20251206_032557", + "iteration": 2, + "trial_index": 1, + "parameters": { + "x1": -3.3329896349459887, + "x2": 11.819649185054004 + }, + "objective_value": 1.4197303445112155, + "notes": "", + "timestamp": "2025-12-06 03:29:42.021000" + }, + { + "_id": "6933a36c40ff909521ebc0c3", + "experiment_id": "exp_20251206_032557", + "iteration": 3, + "trial_index": 2, + "parameters": { + "x1": 0.47781798522919416, + "x2": 5.299124848097563 + }, + "objective_value": 18.527586163986598, + "notes": "", + "timestamp": "2025-12-06 03:30:52.792000" + }, + { + "_id": "6933a38b40ff909521ebc0c4", + "experiment_id": "exp_20251206_032557", + "iteration": 4, + "trial_index": 3, + "parameters": { + "x1": 6.1517495242878795, + "x2": 8.056453275494277 + }, + "objective_value": 67.93869624991603, + "notes": "", + "timestamp": "2025-12-06 03:31:23.019000" + }, + { + "_id": "6933a3aa40ff909521ebc0c5", + "experiment_id": "exp_20251206_032557", + "iteration": 5, + "trial_index": 4, + "parameters": { + "x1": 3.733815406449139, + "x2": 7.317014294676483 + }, + "objective_value": 31.828942575880458, + "notes": "", + "timestamp": "2025-12-06 03:31:54.853000" + } + ] +} \ No newline at end of file diff --git a/scripts/prefect_scripts/experiment_data/bo_experiments.json b/scripts/prefect_scripts/experiment_data/bo_experiments.json new file mode 100644 index 00000000..9d8d25b3 --- /dev/null +++ b/scripts/prefect_scripts/experiment_data/bo_experiments.json @@ -0,0 +1,16 @@ +[ + { + "_id": "6933a24540ff909521ebc0c0", + "experiment_id": "exp_20251206_032557", + "created_at": "2025-12-06 03:25:57.991000", + "n_iterations": 5, + "random_seed": 42, + "status": "completed", + "best_parameters": { + "x1": -3.3329896349459887, + "x2": 11.819649185054004 + }, + "best_value": 1.4197303445112155, + "completed_at": "2025-12-06 03:31:54.966000" + } +] \ No newline at end of file diff --git a/scripts/prefect_scripts/experiment_data/bo_trials.json b/scripts/prefect_scripts/experiment_data/bo_trials.json new file mode 100644 index 00000000..471dbf07 --- /dev/null +++ b/scripts/prefect_scripts/experiment_data/bo_trials.json @@ -0,0 +1,67 @@ +[ + { + "_id": "6933a2fc40ff909521ebc0c1", + "experiment_id": "exp_20251206_032557", + "iteration": 1, + "trial_index": 0, + "parameters": { + "x1": 9.962700307369232, + "x2": 1.565495878458023 + }, + "objective_value": 3.715720823042817, + "notes": "", + "timestamp": "2025-12-06 03:29:00.946000" + }, + { + "_id": "6933a32640ff909521ebc0c2", + "experiment_id": "exp_20251206_032557", + "iteration": 2, + "trial_index": 1, + "parameters": { + "x1": -3.3329896349459887, + "x2": 11.819649185054004 + }, + "objective_value": 1.4197303445112155, + "notes": "", + "timestamp": "2025-12-06 03:29:42.021000" + }, + { + "_id": "6933a36c40ff909521ebc0c3", + "experiment_id": "exp_20251206_032557", + "iteration": 3, + "trial_index": 2, + "parameters": { + "x1": 0.47781798522919416, + "x2": 5.299124848097563 + }, + "objective_value": 18.527586163986598, + "notes": "", + "timestamp": "2025-12-06 03:30:52.792000" + }, + { + "_id": "6933a38b40ff909521ebc0c4", + "experiment_id": "exp_20251206_032557", + "iteration": 4, + "trial_index": 3, + "parameters": { + "x1": 6.1517495242878795, + "x2": 8.056453275494277 + }, + "objective_value": 67.93869624991603, + "notes": "", + "timestamp": "2025-12-06 03:31:23.019000" + }, + { + "_id": "6933a3aa40ff909521ebc0c5", + "experiment_id": "exp_20251206_032557", + "iteration": 5, + "trial_index": 4, + "parameters": { + "x1": 3.733815406449139, + "x2": 7.317014294676483 + }, + "objective_value": 31.828942575880458, + "notes": "", + "timestamp": "2025-12-06 03:31:54.853000" + } +] \ No newline at end of file diff --git a/scripts/prefect_scripts/requirements.txt b/scripts/prefect_scripts/requirements.txt new file mode 100644 index 00000000..699bd67c --- /dev/null +++ b/scripts/prefect_scripts/requirements.txt @@ -0,0 +1,5 @@ +prefect +prefect-slack +ax-platform<2 +pymongo +requests diff --git a/scripts/prefect_scripts/create_deployment.py b/scripts/prefect_scripts/sample_scripts/create_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_deployment.py diff --git a/scripts/prefect_scripts/create_hello_world_deployment.py b/scripts/prefect_scripts/sample_scripts/create_hello_world_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_hello_world_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_hello_world_deployment.py diff --git a/scripts/prefect_scripts/create_pause_deployment.py b/scripts/prefect_scripts/sample_scripts/create_pause_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_pause_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_pause_deployment.py diff --git a/scripts/prefect_scripts/create_pause_slack_deployment.py b/scripts/prefect_scripts/sample_scripts/create_pause_slack_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_pause_slack_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_pause_slack_deployment.py diff --git a/scripts/prefect_scripts/create_sample_transfer_deployment.py b/scripts/prefect_scripts/sample_scripts/create_sample_transfer_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_sample_transfer_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_sample_transfer_deployment.py diff --git a/scripts/prefect_scripts/create_suspend_slack_deployment.py b/scripts/prefect_scripts/sample_scripts/create_suspend_slack_deployment.py similarity index 100% rename from scripts/prefect_scripts/create_suspend_slack_deployment.py rename to scripts/prefect_scripts/sample_scripts/create_suspend_slack_deployment.py diff --git a/scripts/prefect_scripts/hello_world.py b/scripts/prefect_scripts/sample_scripts/hello_world.py similarity index 100% rename from scripts/prefect_scripts/hello_world.py rename to scripts/prefect_scripts/sample_scripts/hello_world.py diff --git a/scripts/prefect_scripts/my_gh_pause_slack_workflow.py b/scripts/prefect_scripts/sample_scripts/my_gh_pause_slack_workflow.py similarity index 100% rename from scripts/prefect_scripts/my_gh_pause_slack_workflow.py rename to scripts/prefect_scripts/sample_scripts/my_gh_pause_slack_workflow.py diff --git a/scripts/prefect_scripts/my_gh_pause_workflow.py b/scripts/prefect_scripts/sample_scripts/my_gh_pause_workflow.py similarity index 100% rename from scripts/prefect_scripts/my_gh_pause_workflow.py rename to scripts/prefect_scripts/sample_scripts/my_gh_pause_workflow.py diff --git a/scripts/prefect_scripts/my_gh_sample_transfer_workflow.py b/scripts/prefect_scripts/sample_scripts/my_gh_sample_transfer_workflow.py similarity index 100% rename from scripts/prefect_scripts/my_gh_sample_transfer_workflow.py rename to scripts/prefect_scripts/sample_scripts/my_gh_sample_transfer_workflow.py diff --git a/scripts/prefect_scripts/my_gh_suspend_slack_workflow.py b/scripts/prefect_scripts/sample_scripts/my_gh_suspend_slack_workflow.py similarity index 100% rename from scripts/prefect_scripts/my_gh_suspend_slack_workflow.py rename to scripts/prefect_scripts/sample_scripts/my_gh_suspend_slack_workflow.py diff --git a/scripts/prefect_scripts/my_gh_workflow.py b/scripts/prefect_scripts/sample_scripts/my_gh_workflow.py similarity index 100% rename from scripts/prefect_scripts/my_gh_workflow.py rename to scripts/prefect_scripts/sample_scripts/my_gh_workflow.py diff --git a/scripts/prefect_scripts/run_gh_pause_slack.py b/scripts/prefect_scripts/sample_scripts/run_gh_pause_slack.py similarity index 100% rename from scripts/prefect_scripts/run_gh_pause_slack.py rename to scripts/prefect_scripts/sample_scripts/run_gh_pause_slack.py diff --git a/scripts/prefect_scripts/run_input_example.py b/scripts/prefect_scripts/sample_scripts/run_input_example.py similarity index 100% rename from scripts/prefect_scripts/run_input_example.py rename to scripts/prefect_scripts/sample_scripts/run_input_example.py diff --git a/scripts/prefect_scripts/run_my_gh_sample_transfer.py b/scripts/prefect_scripts/sample_scripts/run_my_gh_sample_transfer.py similarity index 100% rename from scripts/prefect_scripts/run_my_gh_sample_transfer.py rename to scripts/prefect_scripts/sample_scripts/run_my_gh_sample_transfer.py diff --git a/scripts/prefect_scripts/trigger_flow_programatically_1.py b/scripts/prefect_scripts/sample_scripts/trigger_flow_programatically_1.py similarity index 100% rename from scripts/prefect_scripts/trigger_flow_programatically_1.py rename to scripts/prefect_scripts/sample_scripts/trigger_flow_programatically_1.py diff --git a/scripts/prefect_scripts/trigger_flow_programatically_2.py b/scripts/prefect_scripts/sample_scripts/trigger_flow_programatically_2.py similarity index 100% rename from scripts/prefect_scripts/trigger_flow_programatically_2.py rename to scripts/prefect_scripts/sample_scripts/trigger_flow_programatically_2.py diff --git a/scripts/prefect_scripts/use_flow_directly.py b/scripts/prefect_scripts/sample_scripts/use_flow_directly.py similarity index 100% rename from scripts/prefect_scripts/use_flow_directly.py rename to scripts/prefect_scripts/sample_scripts/use_flow_directly.py