Rename src folders and fix error in CD Pipelines

.azure-pipelines/docker_taxi_cd_pipeline.yml

@@ -8,7 +8,7 @@ trigger:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/docker_taxi/*
-      - src/docker_taxi_src/*
+      - src/docker_taxi/*
       - model/docker_taxi/*
       - src/shared/*
 

.azure-pipelines/docker_taxi_ci_pipeline.yml

@@ -8,7 +8,7 @@ pr:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/docker_taxi/*
-      - src/docker_taxi_src/*
+      - src/docker_taxi/*
       - model/docker_taxi/*
       - src/shared/*
 pool:

.azure-pipelines/london_taxi_cd_pipeline.yml

@@ -8,7 +8,7 @@ trigger:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/london_taxi/*
-      - src/london_src/*
+      - src/london_taxi/*
       - model/london_taxi/*
       - src/shared/*
 

.azure-pipelines/london_taxi_ci_pipeline.yml

@@ -8,7 +8,7 @@ pr:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/london_taxi/*
-      - src/london_src/*
+      - src/london_taxi/*
       - model/london_taxi/*
       - src/shared/*
 

.azure-pipelines/nyc_taxi_cd_pipeline.yml

@@ -8,7 +8,7 @@ trigger:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/nyc_taxi/*
-      - src/nyc_src/*
+      - src/nyc_taxi/*
       - model/nyc_taxi/*
       - src/shared/*
 

.azure-pipelines/nyc_taxi_ci_pipeline.yml

@@ -8,7 +8,7 @@ pr:
       - .azure-pipelines/*
       - mlops/common/*
       - mlops/nyc_taxi/*
-      - src/nyc_src/*
+      - src/nyc_taxi/*
       - model/nyc_taxi/*
       - src/shared/*
 pool:

.github/workflows/docker_taxi_cd_pipeline.yml

@@ -8,7 +8,7 @@ on:
       - mlops/common/**
       - mlops/docker_taxi/**
       - model/docker_taxi/**
-      - src/docker_taxi_src/**
+      - src/docker_taxi/**
       - test/docker_taxi/**
   workflow_call:
     inputs:
@@ -24,7 +24,8 @@ on:
         default: "docker_taxi"
 permissions:
   id-token: write
-  contents: read          
+  contents: read
+
 jobs:
   run-cd-workflow:
     uses: ./.github/workflows/platform_cd_workflow.yml

.github/workflows/docker_taxi_ci_pipeline.yml

@@ -9,7 +9,7 @@ on:
       - 'mlops/common/**'
       - 'mlops/docker_taxi/**'
       - 'model/docker_taxi/**'
-      - 'src/docker_taxi_src/**'
+      - 'src/docker_taxi/**'
       - 'test/docker_taxi/**'
   workflow_call:
     inputs:

.github/workflows/london_taxi_cd_pipeline.yml

@@ -7,7 +7,7 @@ on:
       - mlops/common/**
       - mlops/london_taxi/**
       - model/london_taxi/**
-      - src/london_src/**
+      - src/london_taxi/**
       - test/london_taxi/**
   workflow_call:
     inputs:

.github/workflows/london_taxi_ci_pipeline.yml

@@ -9,7 +9,7 @@ on:
       - 'mlops/common/**'
       - 'mlops/london_taxi/**'
       - 'model/london_taxi/**'
-      - 'src/london_src/**'
+      - 'src/london_taxi/**'
       - 'test/london_taxi/**'
   workflow_call:
     inputs:

.github/workflows/nyc_taxi_cd_pipeline.yml

@@ -8,7 +8,7 @@ on:
       - mlops/common/**
       - mlops/nyc_taxi/**
       - model/nyc_taxi/**
-      - src/nyc_src/**
+      - src/nyc_taxi/**
       - test/nyc_taxi/**
   workflow_call:
     inputs:

.github/workflows/nyc_taxi_ci_pipeline.yml

@@ -9,7 +9,7 @@ on:
       - 'mlops/common/**'
       - 'mlops/nyc_taxi/**'
       - 'model/nyc_taxi/**'
-      - 'src/nyc_src/**'
+      - 'src/nyc_taxi/**'
       - 'test/nyc_taxi/**'
   workflow_call:
     inputs:

.github/workflows/platform_cd_workflow.yml

@@ -94,6 +94,7 @@ jobs:
         shell: bash
         run: |
             [ -d "src/${{ inputs.model_type }}/common/" ] && cp -r src/${{ inputs.model_type }}/common/* model/${{ inputs.model_type }}/online
+        continue-on-error: true
 
       - name: Provision azureml online endpoint
         uses: ./.github/actions/execute_shell_code
@@ -152,6 +153,7 @@ jobs:
         shell: bash
         run: |
             [ -d "src/${{ inputs.model_type }}/common/" ] && cp -r src/${{ inputs.model_type }}/common/* model/${{ inputs.model_type }}/batch
+        continue-on-error: true
 
       - name: Provision azureml batch endpoint
         uses: ./.github/actions/execute_shell_code

.github/workflows/sequence_model_cd_pipeline.yml

@@ -1,4 +1,5 @@
 name: Sequence Model CD Workflow
+
 on:
   push:
     branches:
@@ -24,7 +25,8 @@ on:
         default: "sequence_model"
 permissions:
   id-token: write
-  contents: read         
+  contents: read
+
 jobs:
   run-ci-workflow:
     uses: ./.github/workflows/platform_cd_workflow.yml

README.md

@@ -186,7 +186,7 @@ changed. By defining these paths, we ensure that only the necessary pipelines ar
 tests, and speeding up the overall CI/CD process. For example:
 
 - Changes to the `src/` directory trigger the build and validation pipeline.
-- Changes to the `src/london_src/*` directory trigger the london_taxi pipelines and not the nyc_taxi or docker_taxi pipelines.
+- Changes to the `src/london_taxi/*` directory trigger the london_taxi pipelines and not the nyc_taxi or docker_taxi pipelines.
 
 ### Notes
 

docs/design-decisions/ModelPromotion.md

@@ -57,7 +57,7 @@ on:
       - 'mlops/common/**'
       - 'mlops/nyc_taxi/**'
       - 'model/nyc_taxi/**'
-      - 'src/nyc_src/**'
+      - 'src/nyc_taxi/**'
       - 'test/nyc_taxi/**'
     paths-ignore:
       - '.experimentation/**'

docs/how-to/DSScriptInputsAndOutputs.md

@@ -2,7 +2,7 @@
 
 ## Context and Scope
 
-The GitHub workflows create Azure Machine Learning (AML) jobs which in turn run the scripts defined under this repo's src directory.  For example, [src/nyc_src/train/train.py](/src/nyc_src/train/train.py).  Each of these scripts requires a series of inputs and produces outputs which can be used in downstream scripts.  Understanding how inputs and outputs to these scripts function is critical when integrating promoted models.
+The GitHub workflows create Azure Machine Learning (AML) jobs which in turn run the scripts defined under this repo's src directory.  For example, [src/nyc_taxi/train/train.py](/src/nyc_taxi/train/train.py).  Each of these scripts requires a series of inputs and produces outputs which can be used in downstream scripts.  Understanding how inputs and outputs to these scripts function is critical when integrating promoted models.
 
 ## Script Inputs and Outputs
 
@@ -38,7 +38,7 @@ The resulting command looks like:
 ```yaml
 ...
 command: >-
-  python -m src.nyc_src.train.train
+  python -m src.nyc_taxi.train.train
   --train_path ${{inputs.train_path}}
   --val_path ${{inputs.val_path}}
   --meta_path ${{outputs.meta_path}}

mlops/docker_taxi/components/predict.yml

@@ -14,7 +14,7 @@ outputs:
 environment:  azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.docker_taxi_src.predict.predict
+  python -m src.docker_taxi.predict.predict
   --model_input ${{inputs.model_input}} 
   --test_data ${{inputs.test_data}}
   --predictions ${{outputs.predictions}}

mlops/docker_taxi/components/prep.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.docker_taxi_src.prep.prep 
+  python -m src.docker_taxi.prep.prep 
   --raw_data ${{inputs.raw_data}} 
   --prep_data ${{outputs.prep_data}}
 

mlops/docker_taxi/components/register.yml

@@ -15,7 +15,7 @@ inputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.docker_taxi_src.register.register 
+  python -m src.docker_taxi.register.register 
   --model_metadata ${{inputs.model_metadata}}  
   --model_name ${{inputs.model_name}}
   --score_report ${{inputs.score_report}}

mlops/docker_taxi/components/score.yml

@@ -14,7 +14,7 @@ outputs:
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.docker_taxi_src.score.score 
+  python -m src.docker_taxi.score.score 
   --predictions ${{inputs.predictions}} 
   --model ${{inputs.model}} 
   --score_report ${{outputs.score_report}}

mlops/docker_taxi/components/train.yml

@@ -16,7 +16,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.docker_taxi_src.train.train
+  python -m src.docker_taxi.train.train
   --training_data ${{inputs.training_data}} 
   --test_data ${{outputs.test_data}} 
   --model_output ${{outputs.model_output}}

mlops/docker_taxi/components/transform.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.docker_taxi_src.transform.transform
+  python -m src.docker_taxi.transform.transform
   --clean_data ${{inputs.clean_data}} 
   --transformed_data ${{outputs.transformed_data}}
 

mlops/london_taxi/components/predict.yml

@@ -14,7 +14,7 @@ outputs:
 environment:  azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.london_src.predict.predict
+  python -m src.london_taxi.predict.predict
   --model_input ${{inputs.model_input}} 
   --test_data ${{inputs.test_data}}
   --predictions ${{outputs.predictions}}

mlops/london_taxi/components/prep.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.london_src.prep.prep 
+  python -m src.london_taxi.prep.prep 
   --raw_data ${{inputs.raw_data}} 
   --prep_data ${{outputs.prep_data}}
 

mlops/london_taxi/components/register.yml

@@ -15,7 +15,7 @@ inputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.london_src.register.register 
+  python -m src.london_taxi.register.register 
   --model_metadata ${{inputs.model_metadata}}  
   --model_name ${{inputs.model_name}}
   --score_report ${{inputs.score_report}}

mlops/london_taxi/components/score.yml

@@ -14,7 +14,7 @@ outputs:
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.london_src.score.score 
+  python -m src.london_taxi.score.score 
   --predictions ${{inputs.predictions}} 
   --model ${{inputs.model}} 
   --score_report ${{outputs.score_report}}

mlops/london_taxi/components/train.yml

@@ -16,7 +16,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.london_src.train.train
+  python -m src.london_taxi.train.train
   --training_data ${{inputs.training_data}} 
   --test_data ${{outputs.test_data}} 
   --model_output ${{outputs.model_output}}

mlops/london_taxi/components/transform.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.1-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.london_src.transform.transform
+  python -m src.london_taxi.transform.transform
   --clean_data ${{inputs.clean_data}} 
   --transformed_data ${{outputs.transformed_data}}
 

mlops/nyc_taxi/components/predict.yml

@@ -14,7 +14,7 @@ outputs:
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.nyc_src.predict.predict 
+  python -m src.nyc_taxi.predict.predict 
   --model_input ${{inputs.model_input}} 
   --test_data ${{inputs.test_data}}
   --predictions ${{outputs.predictions}}

mlops/nyc_taxi/components/prep.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.nyc_src.prep.prep 
+  python -m src.nyc_taxi.prep.prep 
   --raw_data ${{inputs.raw_data}} 
   --prep_data ${{outputs.prep_data}}
 

mlops/nyc_taxi/components/register.yml

@@ -15,7 +15,7 @@ inputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.nyc_src.register.register 
+  python -m src.nyc_taxi.register.register 
   --model_metadata ${{inputs.model_metadata}}  
   --model_name ${{inputs.model_name}}
   --score_report ${{inputs.score_report}}

mlops/nyc_taxi/components/score.yml

@@ -14,7 +14,7 @@ outputs:
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 code: ./../../../
 command: >-
-  python -m src.nyc_src.score.score 
+  python -m src.nyc_taxi.score.score 
   --predictions ${{inputs.predictions}} 
   --model ${{inputs.model}} 
   --score_report ${{outputs.score_report}}

mlops/nyc_taxi/components/train.yml

@@ -16,7 +16,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.nyc_src.train.train
+  python -m src.nyc_taxi.train.train
   --training_data ${{inputs.training_data}} 
   --test_data ${{outputs.test_data}} 
   --model_output ${{outputs.model_output}}

mlops/nyc_taxi/components/transform.yml

@@ -12,7 +12,7 @@ outputs:
 code: ./../../../
 environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest
 command: >-
-  python -m src.nyc_src.transform.transform
+  python -m src.nyc_taxi.transform.transform
   --clean_data ${{inputs.clean_data}} 
   --transformed_data ${{outputs.transformed_data}}