Lorrin/nyc taxi update 20251201

.azure-pipelines/requirements/build_validation_requirements.txt

@@ -5,9 +5,9 @@ pytest-cov==3.0.0
 pytest-azurepipelines==1.0.3
 pytest-mock==3.7.0
 pytest==7.1.2
-mlflow==2.14.1
+mlflow-skinny==2.14.3
 mldesigner==0.1.0b4
-azure-ai-ml==1.8.0
+azure-ai-ml>=1.30.0
 azure-identity==1.16.1
 python-dotenv>=0.10.3
-azureml-mlflow>=1.51
+azureml-mlflow==1.60.0

.azure-pipelines/requirements/execute_job_requirements.txt

@@ -1,6 +1,8 @@
 azure-cli==2.64.0
-azure-ai-ml==1.12.1
+azure-ai-ml>=1.30.0
 azure-identity==1.16.1
-mlflow==2.14.1
+mlflow-skinny==2.14.3
+mldesigner==0.1.0b4
+# mlflow-skinny==2.14.3
 python-dotenv>=0.10.3
-azureml-mlflow>=1.51
+azureml-mlflow==1.60.0

.azure-pipelines/templates/build_validation_pipeline.yml

@@ -12,7 +12,7 @@ parameters:
 
 jobs:
 - job: Build_Validation_Pipeline_Docker
-  condition: eq(${{ parameters.is_docker }}, true)
+  condition: eq(${{ parameters.is_docker }}, True)
   steps:
   - script: |
       python -m pip install --upgrade pip
@@ -47,12 +47,12 @@ jobs:
       summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
 
 - job: Build_Validation_Pipeline
-  condition: eq(${{ parameters.is_docker }}, false)
+  condition: eq(${{ parameters.is_docker }}, False)
   steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.9'
+    displayName: 'Use Python 3.10'
     inputs:
-      versionSpec: '3.9'
+      versionSpec: '3.10'
       disableDownloadFromRegistry: true
 
   - task: Cache@2

.azure-pipelines/templates/configure_azureml_agent.yml

@@ -1,8 +1,8 @@
 steps:
 - task: UsePythonVersion@0
-  displayName: 'Use Python 3.9'
+  displayName: 'Use Python 3.10'
   inputs:
-    versionSpec: '3.9'
+    versionSpec: '3.10'
 
 - task: Cache@2
   inputs:

.devcontainer/base/requirements.txt

@@ -1,15 +1,18 @@
-azure-ai-ml>=1.10.0
+azure-ai-ml>=1.30.0
 azure-cli==2.64.0
 azure-identity>=1.15.0
 azure-keyvault-secrets>=4.7.0
-azureml-inference-server-http==0.7.6
-azureml-mlflow>=1.53
+# Bump to a version compatible with psutil>=5.9 (required by azure-cli-core)
+azureml-inference-server-http>=0.8.0,<1.0.0
+azureml-mlflow==1.60.0
 flake8-docstrings==1.7.0
 flake8==6.1.0
 flake8-docstrings==1.7.0
 isort==5.13.2
-mlflow==2.14.1
+mlflow-skinny==2.14.3
 pandas==2.2.3
+numpy==1.26.4; python_version < "3.11"
+numpy==2.3.5; python_version >= "3.11"
 pep8-naming==0.13.3
 pre-commit==3.8.0
 pytest==7.1.2

.github/actions/configure_azureml_agent/action.yml

@@ -5,7 +5,7 @@ description: Installs the required python packages for the workflow.
 inputs:
   versionSpec:
     description: "The Python version to use in the environment."
-    default: "3.9"
+    default: "3.10"
   arm_client_id:
     description: "client id of service principal used to authenticate with azure"
     required: true

.github/requirements/build_validation_requirements.txt

@@ -5,9 +5,10 @@ pytest-cov==3.0.0
 pytest-azurepipelines==1.0.3
 pytest-mock==3.7.0
 pytest==7.1.2
-mlflow==2.11.3
+mlflow-skinny==2.14.3
+# mlflow-skinny==2.14.3
 mldesigner==0.1.0b4
-azure-ai-ml==1.8.0
+azure-ai-ml>=1.30.0
 azure-identity==1.16.1
 python-dotenv>=0.10.3
-azureml-mlflow>=1.51
+# azureml-mlflow will be installed from Microsoft feed in workflow

.github/requirements/execute_job_requirements.txt

@@ -1,8 +1,9 @@
-azure-cli==2.64.0
-azure-ai-ml==1.12.1
+azure-ai-ml==1.30.0
 azure-identity==1.16.1
-mlflow==2.11.3
+mlflow-skinny==2.14.3
+# mlflow-skinny==2.14.3
+mldesigner==0.1.0b4
 python-dotenv>=0.10.3
-azureml-mlflow>=1.51
 azureml-core
-azureml-mlflow>=1.51
+marshmallow==3.20.2
+azureml-mlflow==1.60.0

.github/workflows/build_validation_workflow.yml

@@ -18,15 +18,27 @@ on:
         default: false
 
 env:
-  PYTHON_VERSION: "3.9"
+  PYTHON_VERSION: "3.10"
 
 jobs:
   run-unit-tests-docker:
-    if: ${{ inputs.is_docker == true }}
+    if: ${{ inputs.is_docker }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+          clean: true
+          submodules: false
+
+
+      - name: Show commit and branch
+        run: |
+          echo "GITHUB_SHA=$GITHUB_SHA"
+          git rev-parse HEAD
+          git --no-pager log -1 --oneline --decorate
+
 
       - uses: actions/setup-python@v5
         with:
@@ -37,10 +49,34 @@ jobs:
         run: |
           set -e
           python -m pip install --upgrade pip
-          python -m pip install --upgrade -r .github/requirements/build_validation_requirements.txt      
+          python -m pip install --upgrade \
+            -r .github/requirements/build_validation_requirements.txt \
+            --extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-cli-v2/azureml-mlflow/ \
+            --trusted-host azuremlsdktestpypi.azureedge.net
+
+      - name: Show Package Versions
+        shell: bash
+        run: |
+          python -m pip show mlflow-skinny azureml-mlflow || true
+          python -c "import importlib.metadata as m; print('mlflow-skinny:', m.version('mlflow-skinny'))" || true
+          python -c "import importlib.metadata as m; print('azureml-mlflow:', m.version('azureml-mlflow'))" || true
+
       - name: Lint Code
         run: flake8
 
+      - name: Free up disk space
+        shell: bash
+        run: |
+          echo "Available disk space before cleanup:"
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo docker system prune -af
+          echo "Available disk space after cleanup:"
+          df -h
+
       - name: Build Docker container
         shell: bash
         run: |
@@ -54,7 +90,9 @@ jobs:
           docker run --rm --mount type=bind,src=.,dst=/workspace ${{ inputs.image_name }} /bin/bash -c "
             sudo chown -R vscode /workspace &&
             python -m pip install --upgrade pip &&
-            pip install -r .github/requirements/build_validation_requirements.txt &&
+            pip install -r .github/requirements/build_validation_requirements.txt \
+              --extra-index-url https://azuremlsdktestpypi.azureedge.net/sdk-cli-v2/azureml-mlflow/ \
+              --trusted-host azuremlsdktestpypi.azureedge.net &&
             python -m pytest --junitxml=junit/test-results.xml --cov=. --cov-report=xml
           "
 
@@ -65,11 +103,13 @@ jobs:
           path: "**/test-*.xml"
 
   run-unit-tests:
-    if: ${{ inputs.is_docker == false }}
+    if: ${{ !inputs.is_docker }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.sha }}
 
       - uses: actions/setup-python@v5
         with:
@@ -80,7 +120,23 @@ jobs:
         run: |
           set -e
           python -m pip install --upgrade pip
-          python -m pip install --upgrade -r .github/requirements/build_validation_requirements.txt      
+          python -m pip install --upgrade --no-cache-dir \
+            -r .github/requirements/build_validation_requirements.txt
+
+      - name: Show Package Versions
+        shell: bash
+        run: |
+          python -m pip show mlflow-skinny azureml-mlflow || true
+          python -c "import importlib.metadata as m; print('mlflow-skinny:', m.version('mlflow-skinny'))" || true
+          python -c "import importlib.metadata as m; print('azureml-mlflow:', m.version('azureml-mlflow'))" || true
+
+      - name: Preview updated smoke script
+        shell: bash
+        run: |
+          git rev-parse HEAD
+          git --no-pager log -1 --oneline --decorate
+          sed -n '1,120p' scripts/mlflow_compat_smoke.py || true
+
       - name: Lint Code
         run: flake8
       - name: Run Unit Tests

.github/workflows/cleanup_model_artifacts.yml

@@ -19,7 +19,7 @@ on:
 env:
   CLEAN_UP_DATE: ${{ inputs.date }}
   JOB_TYPE: ${{ inputs.jobType }}
-  PYTHON_VERSION: "3.9"
+  PYTHON_VERSION: "3.10"
   RESOURCE_GROUP_NAME: ${{ vars.RESOURCE_GROUP_NAME }}
   SUBSCRIPTION_ID: ${{ vars.SUBSCRIPTION_ID }}
   WORKSPACE_NAME: ${{ vars.WORKSPACE_NAME }}